{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模拟实现梯度下降法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用梯度下降法求损失函数$J(x)=(x-2)^2 - 1$的最小值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1.  , -0.95, -0.9 , -0.85, -0.8 , -0.75, -0.7 , -0.65, -0.6 ,\n",
       "       -0.55, -0.5 , -0.45, -0.4 , -0.35, -0.3 , -0.25, -0.2 , -0.15,\n",
       "       -0.1 , -0.05,  0.  ,  0.05,  0.1 ,  0.15,  0.2 ,  0.25,  0.3 ,\n",
       "        0.35,  0.4 ,  0.45,  0.5 ,  0.55,  0.6 ,  0.65,  0.7 ,  0.75,\n",
       "        0.8 ,  0.85,  0.9 ,  0.95,  1.  ,  1.05,  1.1 ,  1.15,  1.2 ,\n",
       "        1.25,  1.3 ,  1.35,  1.4 ,  1.45,  1.5 ,  1.55,  1.6 ,  1.65,\n",
       "        1.7 ,  1.75,  1.8 ,  1.85,  1.9 ,  1.95,  2.  ,  2.05,  2.1 ,\n",
       "        2.15,  2.2 ,  2.25,  2.3 ,  2.35,  2.4 ,  2.45,  2.5 ,  2.55,\n",
       "        2.6 ,  2.65,  2.7 ,  2.75,  2.8 ,  2.85,  2.9 ,  2.95,  3.  ,\n",
       "        3.05,  3.1 ,  3.15,  3.2 ,  3.25,  3.3 ,  3.35,  3.4 ,  3.45,\n",
       "        3.5 ,  3.55,  3.6 ,  3.65,  3.7 ,  3.75,  3.8 ,  3.85,  3.9 ,\n",
       "        3.95,  4.  ,  4.05,  4.1 ,  4.15,  4.2 ,  4.25,  4.3 ,  4.35,\n",
       "        4.4 ,  4.45,  4.5 ,  4.55,  4.6 ,  4.65,  4.7 ,  4.75,  4.8 ,\n",
       "        4.85,  4.9 ,  4.95,  5.  ,  5.05,  5.1 ,  5.15,  5.2 ,  5.25,\n",
       "        5.3 ,  5.35,  5.4 ,  5.45,  5.5 ,  5.55,  5.6 ,  5.65,  5.7 ,\n",
       "        5.75,  5.8 ,  5.85,  5.9 ,  5.95,  6.  ])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_x = np.linspace(-1, 6, 141)\n",
    "plot_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([11.25  , 10.9025, 10.56  , 10.2225,  9.89  ,  9.5625,  9.24  ,\n",
       "        8.9225,  8.61  ,  8.3025,  8.    ,  7.7025,  7.41  ,  7.1225,\n",
       "        6.84  ,  6.5625,  6.29  ,  6.0225,  5.76  ,  5.5025,  5.25  ,\n",
       "        5.0025,  4.76  ,  4.5225,  4.29  ,  4.0625,  3.84  ,  3.6225,\n",
       "        3.41  ,  3.2025,  3.    ,  2.8025,  2.61  ,  2.4225,  2.24  ,\n",
       "        2.0625,  1.89  ,  1.7225,  1.56  ,  1.4025,  1.25  ,  1.1025,\n",
       "        0.96  ,  0.8225,  0.69  ,  0.5625,  0.44  ,  0.3225,  0.21  ,\n",
       "        0.1025,  0.    , -0.0975, -0.19  , -0.2775, -0.36  , -0.4375,\n",
       "       -0.51  , -0.5775, -0.64  , -0.6975, -0.75  , -0.7975, -0.84  ,\n",
       "       -0.8775, -0.91  , -0.9375, -0.96  , -0.9775, -0.99  , -0.9975,\n",
       "       -1.    , -0.9975, -0.99  , -0.9775, -0.96  , -0.9375, -0.91  ,\n",
       "       -0.8775, -0.84  , -0.7975, -0.75  , -0.6975, -0.64  , -0.5775,\n",
       "       -0.51  , -0.4375, -0.36  , -0.2775, -0.19  , -0.0975,  0.    ,\n",
       "        0.1025,  0.21  ,  0.3225,  0.44  ,  0.5625,  0.69  ,  0.8225,\n",
       "        0.96  ,  1.1025,  1.25  ,  1.4025,  1.56  ,  1.7225,  1.89  ,\n",
       "        2.0625,  2.24  ,  2.4225,  2.61  ,  2.8025,  3.    ,  3.2025,\n",
       "        3.41  ,  3.6225,  3.84  ,  4.0625,  4.29  ,  4.5225,  4.76  ,\n",
       "        5.0025,  5.25  ,  5.5025,  5.76  ,  6.0225,  6.29  ,  6.5625,\n",
       "        6.84  ,  7.1225,  7.41  ,  7.7025,  8.    ,  8.3025,  8.61  ,\n",
       "        8.9225,  9.24  ,  9.5625,  9.89  , 10.2225, 10.56  , 10.9025,\n",
       "       11.25  ])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot_y = (plot_x - 2.5) ** 2 - 1\n",
    "plot_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4ldW5/vHvkzmBJBAykJAREmYIYGRWQVAUUNRaxalqnap1qD+rx9r2dNaetp462zpTRVAQFRVHREVkCiBjAoSQOSQhQBIy7531+4PgoZY5O1l7eD7XlYskbnhvQrxZrLXe9YoxBqWUUp7Pz3YApZRSrqGFrpRSXkILXSmlvIQWulJKeQktdKWU8hJa6Eop5SW00JVSyktooSullJfQQldKKS8R0JUXi46ONqmpqV15SaWU8njr1q3ba4yJOdHrurTQU1NTyc7O7spLKqWUxxORwpN5nU65KKWUl9BCV0opL6GFrpRSXkILXSmlvIQWulJKeQktdKWU8hJa6Eop5SU8otA/2FTO3NUntQ1TKaXcSumBRv78YS6VdU2dfi2PKPQlm8v528fbaXY4bUdRSqlT8sbaYv751S5aHG2dfi2PKPQrz0xif0MrH2+tsB1FKaVOmrPNsCC7mLMyYkjsGdbp1/OIQp+YHk1iz1DeWFtkO4pSSp20r3ZUUV7TxFVnJnXJ9Tyi0P38hCuzkliRV01hdb3tOEopdVLmrSkiunsQUwbFdcn1PKLQAS7PSsRPDs1HKaWUu6usbWJpbiU/GJVIUEDXVK3HFHp8ZCiTB8SyYF0JDmfnLy4opVRHLFxfgrPNcGUXTbeABxU6wOzRyVTVNfN5bqXtKEopdUxtbYY31hYzJi2KvjHdu+y6HlXokwfEEBsezHyddlFKubFV+dUUVjcwe3TXjc7Bwwo9wN+PH2Yl8sX2SsprGm3HUUqpo5q3tpiIkAAuHBrfpdf1qEIHuDIrmTYDC7JLbEdRSqn/sL++hY+37OGyUYmEBPp36bU9rtCTe4UxMT2aN9YW09ZmbMdRSql/s2hDKS3Oti5dDD3M4wodYPboJEoPNLI8b6/tKEop9R1jDPPXFJGZ1INB8RFdfn2PLPTzBsfRMyyQ+Wv0zlGllPtYX7SfnZUHu+zO0O/zyEIPDvDnB6MS+XRbBXsPNtuOo5RSAMxfU0xYkD8zMxOsXN8jCx0OTbs42gxvrdPFUaWUfXVNrby/qZyLMxPoHhxgJYPHFnp6bDhZKT15Y20xxujiqFLKrne/LaOx1cns0cnWMnhsocOhO0fz99azKn+f7ShKKR9mjGH+2iIG9g4nMzHSWo4TFrqIvCQilSKy5YjPRYnIpyKys/3Hnp0b8+hmDIsnIiRAn2aklLJqY0kNW0pruWp0MiJiLcfJjNBfAS743uceBJYaYzKApe0fd7nQIH8uPyOJj7fuoapOF0eVUna8tqqQsCB/LhvVx2qOExa6MeYr4PtzGrOAOe3vzwEucXGuk3bN2GRanYY3s/V8F6VU1zvQ0MJ7G8uYNaIP4SGBVrOc7hx6nDGmHKD9x9hjvVBEbhWRbBHJrqqqOs3LHVu/mO6M79eL11cX4dQ7R5VSXWzhuhKaHW1cO9beYuhhnb4oaox5zhiTZYzJiomJ6ZRrXDc2hdIDjSzTY3WVUl2orc0wd3URo5J7MCTB3mLoYadb6BUiEg/Q/qPVJp06OI7Y8GBe08VRpVQX+mZXNbv31nPt2BTbUYDTL/TFwPXt718PvOuaOKcn0N+P2aOT+XJHFUXVDTajKKV8yGurCukZFsj0YV17TO6xnMy2xXnASmCAiJSIyE3An4HzRGQncF77x1ZdNToJPxHmrtFRulKq8+2paeLTnAquyErq8mNyj+WE96caY646xn+a4uIsHRIfGcrUQbEsyC7h/53Xn+AA9/gCK6W807w1hzZiXD3G/mLoYR59p+j3XTs2hX31LXy4eY/tKEopL9bqbGP+2iLO6R9DSq9utuN8x6sKfUK/aFJ7hfHaKp12UUp1nqU5FVTUNrvNYuhhXlXofn7CtWNTyC7cT055re04Sikv9eqqQhIiQzh34DFvwbHCqwod4PIzEgkO8NNRulKqU+RXHWRFXjVXj0nG38/euS1H43WF3iMsiJnDE3hnQyl1Ta224yilvMzc1UUE+AlXWHoq0fF4XaEDXDcuhfoWJ+9sKLUdRSnlRRpbnCzILmba0N7EhofYjvMfvLLQMxMjGdongldXFerDL5RSLvPexjJqmxxcO8a9FkMP88pCFxF+NC6VHRUHWbmr2nYcpZQXMMbw8jcFDIgLZ2zfKNtxjsorCx3g4swEoroF8fI3BbajKKW8wJrd+8gpr+WGCalWH2JxPF5b6CGB/lw9OpnPcioo3qfnuyilOuaVbwroERbIJSPsPsTieLy20OHQnaN+IszRUbpSqgNK9jfw8dY9zD4zmdAg9z1WxKsLvXdkCBcO7c0b2cXUNztsx1FKeahXVxUiIlw3zj0XQw/z6kIHuHFCGnVNDhatL7EdRSnlgRpbnMxfU8y0IXH06RFqO85xeX2hj0ruwfDESF75poA2fUSdUuoUvb2hlJrGVm4Yn2Y7ygl5faGLCDdOSGVXVT1f5+21HUcp5UGMMbzyzW6GJERwZmpP23FOyOsLHWD6sHiiuwfz8ordtqMopTzIyl3V7Kg4yA3j3Xer4pF8otCDA/y5Zkwyy7ZXsXtvve04SikP8dKKAnp1C+KizATbUU6KTxQ6wDVjkwn01y2MSqmTU1TdwNLcCq4ek+w2j5g7EZ8p9NjwEGYOT2BBdrGewqiUOqF/rSzAX4Rr3PTclqPxmUIHuGF8KvUtThau0y2MSqljq2928EZ2MRcOi6d3pPudqngsPlXomUk9GJXcgzm6hVEpdRyL1pdQ1+TgxgmptqOcEp8qdIAbJqRRUN3A57mVtqMopdxQW9uhUxUzEyMZmdTDdpxT4nOFfuHQ3sRHhvDC1/m2oyil3NCy7ZXkV9Xz44lpHrFV8UgdKnQRuVdEtorIFhGZJyJuP9kU6O/HjyeksSp/H5tKDtiOo5RyM899lU+fHqFMHxZvO8opO+1CF5E+wN1AljFmKOAPzHZVsM40e3QS4cEBPL9cbzRSSv2fjcUHWL17HzdOSCXQ3/MmMDqaOAAIFZEAIAwo63ikzhceEshVY5JZsrlcz0pXSn3n+eX5hIcEMHt0su0op+W0C90YUwr8DSgCyoEaY8wnrgrW2W4Yn4oAL68osB1FKeUGivc1sGRzOVePTqZ7cIDtOKelI1MuPYFZQBqQAHQTkWuP8rpbRSRbRLKrqqpOP6mLJfQI5aLMBOavLaKmQW80UsrXvbRiN34i3OBhWxWP1JEpl6nAbmNMlTGmFVgEjP/+i4wxzxljsowxWTExMR24nOvdfFYaDS1OXl9TZDuKUsqimoZW3lhbzMWZCcRHuveZ58fTkUIvAsaKSJgc2tszBchxTayuMSQhkonp0by8YjctjjbbcZRSlsxdU0hDi5Obz+prO0qHdGQOfTWwEFgPbG7/tZ5zUa4uc8vZfamsa2bxRo9Yz1VKuVizw8krKwo4KyOawQkRtuN0SId2uRhjfmOMGWiMGWqMuc4Y0+yqYF3l7IxoBsSF88LyfIzR4wCU8jWLvy2jsq6ZWzx8dA4+eKfo94kIt5zdl9w9dSzfqU80UsqXGGN4fnk+A3uHc1ZGtO04HebzhQ5wcWYCcRHBPL9cjwNQypd8uaOKHRUHueWsvh53m//RaKEDQQF+3DA+jeU797KtrNZ2HKVUF3l+eT5xEcEe80SiE9FCb3f1mGS6BfnrKF0pH7GltIYVedXcOCGNoADvqELv+F24QGRoIFeemczijWV6HIBSPuAfX+6ie3AAV3nobf5Ho4V+hFvOTsNP0FG6Ul4uv+ogH2wu59qxKUSGBtqO4zJa6EeIjwzlB6MSmb+2mMq6JttxlFKd5J9f5hPk78dNE9NsR3EpLfTvue2cfjicbbz0dYHtKEqpTlB2oJFFG0q48swkYsKDbcdxKS3070mL7saM4Qm8tqpQD+1Sygs9vzwfY+DWsz3/RqLv00I/ijsm9eNgs4N/rSywHUUp5ULVB5uZt6aIWSP6kNgzzHYcl9NCP4pB8RFMGRjLSyt209DisB1HKeUiL68ooNnRxu2TvG90Dlrox3TH5HT2N7Qyb02x7ShKKReobWplzsoCLhjSm/TYcNtxOoUW+jGckdKTsX2jeO6rXTQ7nLbjKKU66LVVhdQ1Ofjp5HTbUTqNFvpx/HRyOhW1zSxaX2o7ilKqAxpbnLy4fDfn9I9haJ9I23E6jRb6cUxMj2Z4YiT/+HIXDqc+AEMpT/XG2iKq61u8enQOWujHJSLcMSmdwuoGPthcbjuOUuo0tDjaeO6rfM5M7cnotCjbcTqVFvoJnD84jvTY7jyzbBdtbfoADKU8zTvfllJW08QdXj46By30E/LzE+6Y1I/tFXUsza20HUcpdQqcbYZ/fLGLwfERTOrvXg+p7wxa6CfhoswEUnqF8fjSHfqYOqU8yOKNpeTvreeuc9O94gEWJ6KFfhIC/f24c3I6W0pr+XRbhe04SqmT4HC28cTSPAb2DmfakN6243QJLfSTdOnIPqT2CuOxz3bqKF0pD/Dut2Xs3lvPz6b2x8/P+0fnoIV+0gL8/bjr3Ay2ldfy8VYdpSvlzhzONp78fCeD4yOYNiTOdpwuo4V+CmaNSCAtuhuPfbZDd7wo5cbe+baMguoGfjY1wyfmzg/rUKGLSA8RWSgiuSKSIyLjXBXMHR0apaeTu6eOj7fusR1HKXUUh0fnQxIiOG+w74zOoeMj9MeBj4wxA4FMIKfjkdzbxZkJ9I3uxmOf7dRRulJuaNGGUgqrG/jZ1P4+NTqHDhS6iEQAZwMvAhhjWowxB1wVzF0F+Ptx95QMtlfU8eEWHaUr5U5a20fnw/pEMnVQrO04Xa4jI/S+QBXwsohsEJEXRKSbi3K5tYsyE+gX043Hl+pculLuZNH6Eor3Nfrc3PlhHSn0AGAU8KwxZiRQDzz4/ReJyK0iki0i2VVVVR24nPvw9xPunpLBjoqDesaLUm6ixdHGk5/nkZkYybkDfW90Dh0r9BKgxBizuv3jhRwq+H9jjHnOGJNljMmKifGeW29nDk8gPbY7jy/diVNH6UpZ99b6Ekr2N/rk3Plhp13oxpg9QLGIDGj/1BRgm0tSeQB/P+GeKRnkVR7k/U1ltuMo5dNaHG089XkeI5J6MGmA9wwcT1VHd7ncBcwVkU3ACODhjkfyHDOGxdM/7tAoXc9LV8qeBeuKKT3gu3Pnh3Wo0I0x37ZPpww3xlxijNnvqmCewM9PuHdqf/Kr6lm0QZ9qpJQNTa1Onli6kzNSenKOD5yoeDx6p2gHXTC0N8MTI3n8s5367FGlLJjzTQEVtc08MG2AT4/OQQu9w0SEB6YNpPRAI3NXFdmOo5RPqWls5ZkvdjFpQAxj+vayHcc6LXQXmJgRzfh+vXh6WR4Hmx224yjlM57/Kp+axlZ+fv6AE7/YB2ihu8j90wZQXd/CS1/vth1FKZ9QVdfMSyt2M3N4PEP7RNqO4xa00F1kZHJPzh8cx3Nf5bOvvsV2HKW83tPL8mh2tHGfjs6/o4XuQj+fNoCGFgfPfpFnO4pSXq14XwNzVxdyRVYSadE+ceLISdFCd6H+ceFcOjKROSsLKa9ptB1HKa/198924CeHbu5T/0cL3cXuPS8DDDz6yQ7bUZTyStvKanl7Qyk3jE+ld2SI7ThuRQvdxRJ7hnH9+BTeWl9CTnmt7ThKeZ1HPswhIiSQOyal247idrTQO8GdkzOICAnkkQ9zbUdRyqt8taOK5Tv3cte56USGBdqO43a00DtBZFggd52b3v7N5x1HBitlm7PN8PCSHJKiQrluXIrtOG5JC72TXDcuhaSoUB5ekqvH6yrlAm9vKCV3Tx0PTBtIcIC/7ThuSQu9kwQH+HP/tIHklB9awFFKnb6mViePfrKdzKQezBwebzuO29JC70Qzh8WTmRjJo59sp6lVD+5S6nS9+PVuymuaeOjCgT5/ANfxaKF3Ij8/4RfTB1Fe08SLeiSAUqel+mAz//hiF1MHxekBXCeghd7JxvbtxXmD43hmWR6VtU224yjlcR79dAeNrU4evHCg7ShuTwu9Czw0fRAtzjb+9sl221GU8ig55bXMX1PEdeNSSI/tbjuO29NC7wJp0d24cUIaC9aVsKW0xnYcpTyCMYbfv7eNiNBAvcX/JGmhd5E7z00nKiyI3723FWN0G6NSJ/LJtgpW5lfz/87rT4+wINtxPIIWeheJCAnkvvMHsLZgP0s277EdRym31uxw8vCSHDJiu3P16GTbcTyGFnoXuvLMJAbFR/DwkhzdxqjUcby8ooDC6gZ+PXMwAf5aUydLv1JdyN9P+PXMQZQeaOSF5fm24yjllqrqmnnq8zymDIzl7P4xtuN4FC30Lja+XzTThsTx9LJdema6Ukfxl49yaWp18ssZg2xH8Tha6Bb8asZg2ozhjx/k2I6ilFtZV7ifBetKuGliGn1jdJviqepwoYuIv4hsEJH3XRHIFyRFhXHHpHQ+2FTOiry9tuMo5RacbYb/fncLcRHB3KXbFE+LK0bo9wA61DxFt53Tl+SoMP773S20ONpsx1HKutdXF7K1rJZfzRhM9+AA23E8UocKXUQSgRnAC66J4ztCAv357cWD2VVVz8sr9JwX5duqDzbz14+3M75fLz1NsQM6OkJ/DHgAOOYQU0RuFZFsEcmuqtKHPRzp3IFxTB0Uy+NLd7KnRs95Ub7rLx9tp6HFye8uHqKnKXbAaRe6iMwEKo0x6473OmPMc8aYLGNMVkyMbkH6vv+eOQRHm+FPS3TWSvmm9UX7eSO7mBsnpJIRF247jkfryAh9AnCxiBQA84FzReQ1l6TyIcm9wrhjUj/e21imC6TK5zicbd8thN4ztb/tOB7vtAvdGPMLY0yiMSYVmA18boy51mXJfMhPzulHaq8wfvXOFr2DVPmUOSsL2VKqC6GuovvQ3UBIoD9/vGQYu/fW88yyPNtxlOoSZQcaefST7UwaEKMLoS7ikkI3xnxhjJnpil/LV03MiObSkX149std5FXW2Y6jVKf7zeKttBnDH2YN1YVQF9ERuhv55YxBdAsO4KFFW2hr0yN2lff6aMsePt1Wwb1T+5MUFWY7jtfQQncj0d2DeejCQawp2MeCdcW24yjVKeqaWvnt4q0M7B3Ojyem2Y7jVbTQ3cwPsxIZnRbFw0ty2Xuw2XYcpVzu0U92UFHXxCOXDSNQj8Z1Kf1quhkR4eFLh9LQ4uD3722zHUcpl9pQtJ85Kwu4bmwKI5N72o7jdbTQ3VB6bDh3Ts5g8cYyPt1WYTuOUi7R7HDywMJN9I4I4f5pA2zH8Upa6G7q9kn9GNg7nF++vZmahlbbcZTqsCeX5rGz8iAPXzaM8JBA23G8kha6mwoK8OOvl2dSXd/CHz/QqRfl2baU1vDsl7v4wahEJg+ItR3Ha2mhu7FhiZHcdnZfFqwr4csderCZ8kytzjbuX7iJqG5B/HqmPoWoM2mhu7m7p2SQHtudX7y1ibomnXpRnufZL3aRU17LHy8ZSo+wINtxvJoWupsLCfTnL5cPp7y2iUc+zLUdR6lTsn1PHU9+vpOLMhOYNqS37TheTwvdA4xK7slNE9J4fXURy3fq1IvyDC2ONu5b8C3hIYH89qLBtuP4BC10D/HzaQNIj+3O/Qs26a4X5RGe/HwnW0prefjSYfTqHmw7jk/QQvcQIYH+/P2KEew92Myv391iO45Sx7W+aD9PL8vjB6MSuWCoTrV0FS10DzIsMZK7pxy64ei9jWW24yh1VA0tDu57cyPxkaH85mKdaulKWuge5o5J/chM6sGv3tlCRa0+h1S5n0eW5LJ7bz1/+2EmEXoDUZfSQvcwAf5+/P2KTJodTu5fuAlj9Jhd5T6+3FHFq6sKuWliGuP69bIdx+dooXugvjHdeWj6IL7aUcW/VhbajqMUAPvqW7h/wUYyYrvrWS2WaKF7qOvGpjB5QAx/WpJDTnmt7TjKxxljuH/BRg40tPLY7BGEBPrbjuSTtNA9lIjwtx9mEhkayF3zNtDQ4rAdSfmwV74pYGluJQ9NH8iQhEjbcXyWFroH69U9mMeuHMGuqoP84X09wEvZsbWshkeW5DJ1UCzXj0+1HcenaaF7uAnp0dx+Tj/mrSnmg03ltuMoH9PQ4uCueRvo2S2Qv1yeqQ97tkwL3Qvce15/Rib34MFFmyje12A7jvIhv128ld176/n7lSOI6qYHb9mmhe4FAv39eGL2SDBw17wNtDjabEdSPuCdDaW8mV3CnZPTGd8v2nYcRQcKXUSSRGSZiOSIyFYRuceVwdSpSYoK4y+XD+fb4gM8vCTHdhzl5XZU1PGLRZsZnRbFPVMybMdR7ToyQncA9xljBgFjgZ+KiN7na9GFw+K5aWIar3xTwGI9GkB1koPNDn7y2jq6BQfw1FUjCfDXf+i7i9P+kzDGlBtj1re/XwfkAH1cFUydngcvHEhWSk8efGsTOyvqbMdRXsYYw38t3ERhdQNPXT2S2IgQ25HUEVzyV6uIpAIjgdWu+PXU6Qv09+Opq0cRFuTP7XPXU9+s+9OV67y0ooAPNpdz/7QBjO2rt/a7mw4Xuoh0B94CfmaM+Y9bFkXkVhHJFpHsqip9OENX6B0ZwhOzR5JfdZAHF23W816US6wr3McjS3I4b3Act53d13YcdRQdKnQRCeRQmc81xiw62muMMc8ZY7KMMVkxMTEduZw6BePTo7nv/AG8t7GM55fn246jPNyemiZ+8tp6+vQM5W8/1P3m7qoju1wEeBHIMcb8r+siKVe5Y1I/ZgyL588f5vLF9krbcZSHamp1cuur2TQ0O3j+R1lEhuqRuO6qIyP0CcB1wLki8m3723QX5VIuICL89YfDGdg7grvmbWBX1UHbkZSHMcbw4Fub2Fxaw2OzR9I/Ltx2JHUcHdnl8rUxRowxw40xI9rflrgynOq4sKAAnvvRGQT5+3HLv7KpbdLnkaqT99xX+bzzbRn3ndef8wbH2Y6jTkA3kPqAxJ5hPHvtGRRVN3D3vA0423SRVJ3Ysu2V/PmjXGYMj+enk9Ntx1EnQQvdR4xOi+J3s4bwxfYq/vSB3kmqjm/7njrufn0Dg3pH8NfLh+siqIcIsB1AdZ1rxqSQV3mQl1bsJjkqlBsmpNmOpNxQRW0TN768htAgf164PouwIK0JT6F/Uj7mVzMGU7q/kd+9v42EHqGcP6S37UjKjdQ3O/jxK2upaWzljdvGkdAj1HYkdQp0ysXH+PsJj88eyfA+kdw9fwMbiw/YjqTchMPZxp2vryd3Tx1PXTOKoX30yUOeRgvdBx36p/SZxIQHc9OctXqGusIYw28Wb2XZ9ip+P2sIkwfE2o6kToMWuo+KCQ/m5RtG0+Jo4/qX11B9sNl2JGXR08vymLu6iJ+c049rxqTYjqNOkxa6D0uP7c6LN5xJ2YFGrn95DXW6R90nvbqqkL99soNLR/bhgWkDbMdRHaCF7uPOTI3i2WvOILe8jpvnZNPU6rQdSXWhd78t5b/f3cLUQbH85fLh+Pnp9kRPpoWumDwwlkevyGRNwT7ufH09rU59hJ0vWJZbyX1vbmR0ahRPXT2KQH1QhcfTP0EFwKwRffj9rKF8llPJ/Qs20qZ3k3q1tQX7+Mlr6xgYH84L12cREuhvO5JyAd2Hrr5z3dgUahtb+evH2wn09+N/fqD/BPdG6wr3ccNLa+jTM5Q5N44mPERPT/QWWujq3/x0cjotjjYeX7oTQEvdy6wr3MePXlxDXEQI824ZS6/uwbYjKRfSQlf/4d7z+mOAJ5buRAT+fJmWujdYV7iP619ae6jMbx1LnD4P1OtooaujundqBhjDE5/nAVrqnu5wmceEB2uZezEtdHVUIsK95/UH4InP83A4Df9z+XDdCeGBvtm1l1vmZBPbPs2iZe69tNDVMR0u9UB/Px79dAe1Ta08dfUo3RHhQT7Zuoc7520gtVcYr940Rsvcy+lwSx2XiHDXlAz+MGsIS3Mruf4lvaPUUyxcV8Ltc9czOD6CN28bp2XuA7TQ1Um5blwqj105gnWF+7n6+dV69oube/Hr3fx8wUbG9e3F3JvH0CMsyHYk1QW00NVJmzWiD8//KIudlXVc/o+V7N5bbzuS+p62NsMjS3L4w/vbmD6sNy/ekEW3YJ1Z9RVa6OqUTB4Yy9ybx1DT2Mqlz6xgdX617UiqXUOLg9vnruOfX+Xzo3EpPHnVKIIDdL3Dl2ihq1N2RkoUb98xnqhuQVz74mreWldiO5LPq6ht4sp/ruKTbRX85qLB/H7WUPx1m6nP0UJXpyWlVzfevn0CWSlR3LdgI49+sl3Pf7FkW1ktlzy9gl1VB3n+uixu1GfF+qwOFbqIXCAi20UkT0QedFUo5RkiwwKZ8+PRXJGVyJOf53Hrq9nUNOoOmK70zoZSLnt2BcbAm7eNY+rgONuRlEWnXegi4g88DVwIDAauEpHBrgqmPENQwKFDvH570WC+2F7FxU99zbayWtuxvF6Lo43fvLuFn73xLcP79GDxnRP0GaCqQyP00UCeMSbfGNMCzAdmuSaW8iQiwg0T0njjtrE0tTq57NkVOq/eicprGpn93ErmrCzk5olpzL1lDLG6x1zRsULvAxQf8XFJ++eUjzojJYr37zqLzMQe3LdgI/+1cBP1zQ7bsbzK0pwKZj7xNbl76njq6pH8auZgPY5Bfacj3wlHW0L/j1UxEblVRLJFJLuqqqoDl1OeICY8mLk3j+GOSf14c10xM55Yzoai/bZjebzGFie/emczN83JJiY8mMV3TmDm8ATbsZSb6UihlwBJR3ycCJR9/0XGmOeMMVnGmKyYmJgOXE55igB/Px64YCDzbxlLq9Nw+T9W8vhnO3Hoo+1Oy+aSGmY8uZzXVhVx69l9effOCaTHhtuOpdxQRwp9LZAhImkiEgTMBha7JpbyBmP69mLJPWdx0fB4/v7ZDi7/x0py9+iC6clqanXyv5/u4NJnVtDQ7OT1m8fw0PRBerOQOqbTLnRjjAO4E/ji6VyGAAAH20lEQVQYyAHeNMZsdVUw5R0iQwN5bPZInrhqJEX7Gpj5xNf85aNcmlqdtqO5tVX51Ux/YjlPLN3JzOHxfPSzsxifHm07lnJzYkzX3QySlZVlsrOzu+x6yr3sr2/hT0tyWLiuhJReYfzpkmFMzNCSOtKBhhYeWZLLG9nFJEWF8qdLhnF2f52q9HUiss4Yk3XC12mhq672Td5eHnp7MwXVDUwbEsd/XTCQvjHdbceyqsXRxmurCnni853UNTm45ay+3DMlg9AgnV5RWujKzTW1OnlheT7PfrGLZkcb145N4Z4pGfTs5lvHvBpj+HjrHv78YS4F1Q1MTI/mlzMGMSg+wnY05Ua00JVHqKxr4rHPdjJ/TRHdggO47ey+XDculcjQQNvROpUxhhV51Ty+dAdrC/aTEdudh2YMYlL/GET0UC3177TQlUfZUVHH/3yYy9LcSsKDA7h+fCo/nphGlJeN2I0xLM2p5MlleWwsPkBcRDB3T8ngyqwkAvQGIXUMWujKI20preGZL/L4cMseQgL8mT06iWvHptDPw+fYm1qdvLexjBe/3k3unjqSokK5/Zx0fnBGH92GqE5IC115tLzKOp5Ztov3NpXR6jSM79eLa8akcP6QOI+61X1X1UHmripi4bpiapscpMd2545J/bg4M0FH5OqkaaErr1BV18yb2cW8vrqI0gONRHcPZubweKYPiycrpSd+bvgQh8raJj7csocPNpWzpmAfgf7CtCG9uXZsCmPSonSOXJ0yLXTlVZxthq92VPHG2mKWba+k2dFGbHgw04fFc+7AWM5MjbK2xc8Yw66qepbvrOLDLXtYW7APY6B/XHdmjejDFVlJxIQHW8mmvIMWuvJaB5sdLM2pYMnmcr7YXkWzo40gfz9GJvdgfL9oxvSNYlB8RKftlHG2GXbvPciGogN8s6uab3btpaK2GThU4tOHxTNjWDwZcXreinINLXTlE+qbHawt2MfKXdWs2LWXrWW1HP6WTuwZyqD4CAb1DicxKoz4yBDiI0NJ6BFCWFDAcX9dZ5uhqq6ZsppG9tQ0UXagkV1VB9lWVsv2ijqaWg8dNNarWxDj+vVifL9oxvfrRWp0t87+LSsfpIWufNKBhhY2FB8gp7yWbWW15JTXkr+3nu9/mwf6CyGB/u1vfgT4+dHU6mx/a6PJ4fyPn9MjLJDB8REMio9gcHwEQ/tEkhHb3S3n8ZV3OdlCP/4wRSkP0yMsiMkDYpk8IPa7zzU7nFTUNFNe00h5TRNlNY3UNTlobHHS7HDS2OLE0WYICfQntL3gQ4MCiA0PJqFHCL0jDo3qI0MDdUFTuTUtdOX1ggP8Se4VRnKvMNtRlOpUuhFWKaW8hBa6Ukp5CS10pZTyElroSinlJbTQlVLKS2ihK6WUl9BCV0opL6GFrpRSXqJLb/0XkSqg8DR/ejSw14VxOpsn5dWsnceT8npSVvCsvB3NmmKMiTnRi7q00DtCRLJP5iwDd+FJeTVr5/GkvJ6UFTwrb1dl1SkXpZTyElroSinlJTyp0J+zHeAUeVJezdp5PCmvJ2UFz8rbJVk9Zg5dKaXU8XnSCF0ppdRxeFShi8gPRWSriLSJiFuubovIBSKyXUTyRORB23mOR0ReEpFKEdliO8uJiEiSiCwTkZz274F7bGc6HhEJEZE1IrKxPe/vbGc6ERHxF5ENIvK+7SwnIiIFIrJZRL4VEbd+DJqI9BCRhSKS2/79O66zruVRhQ5sAS4DvrId5GhExB94GrgQGAxcJSKD7aY6rleAC2yHOEkO4D5jzCBgLPBTN//aNgPnGmMygRHABSIy1nKmE7kHyLEd4hRMNsaM8ICti48DHxljBgKZdOLX2KMK3RiTY4zZbjvHcYwG8owx+caYFmA+MMtypmMyxnwF7LOd42QYY8qNMevb36/j0P8UfeymOjZzyMH2DwPb39x2wUpEEoEZwAu2s3gTEYkAzgZeBDDGtBhjDnTW9Tyq0D1AH6D4iI9LcOPS8VQikgqMBFbbTXJ87VMY3wKVwKfGGHfO+xjwANBmO8hJMsAnIrJORG61HeY4+gJVwMvt01kviEi3zrqY2xW6iHwmIluO8ua2I90jHO0Jwm47KvNEItIdeAv4mTGm1nae4zHGOI0xI4BEYLSIDLWd6WhEZCZQaYxZZzvLKZhgjBnFoenNn4rI2bYDHUMAMAp41hgzEqgHOm1tze0eEm2MmWo7QweUAElHfJwIlFnK4nVEJJBDZT7XGLPIdp6TZYw5ICJfcGi9wh0XoCcAF4vIdCAEiBCR14wx11rOdUzGmLL2HytF5G0OTXe649paCVByxL/OFtKJhe52I3QPtxbIEJE0EQkCZgOLLWfyCiIiHJqHzDHG/K/tPCciIjEi0qP9/VBgKpBrN9XRGWN+YYxJNMakcuh79nN3LnMR6SYi4YffB87HPf+ixBizBygWkQHtn5oCbOus63lUoYvIpSJSAowDPhCRj21nOpIxxgHcCXzMoUW7N40xW+2mOjYRmQesBAaISImI3GQ703FMAK4Dzm3fqvZt+4jSXcUDy0RkE4f+ov/UGOP22wE9RBzwtYhsBNYAHxhjPrKc6XjuAua2fy+MAB7urAvpnaJKKeUlPGqErpRS6ti00JVSyktooSullJfQQldKKS+hha6UUl5CC10ppbyEFrpSSnkJLXSllPIS/x+DZnALDR7YeQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(plot_x, plot_y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dJ(theta):\n",
    "    return 2 * (theta - 2.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def J(theta):\n",
    "    return (theta - 2.5) ** 2 - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.499891109642585\n",
      "-0.99999998814289\n"
     ]
    }
   ],
   "source": [
    "# 学习率\n",
    "eta = 0.1\n",
    "\n",
    "epsilon = 1e-8\n",
    "\n",
    "theta = 0.0\n",
    "theta_history = [theta]\n",
    "while True:\n",
    "    gradient = dJ(theta)\n",
    "    # 记录上一次横坐标的值\n",
    "    last_theta = theta\n",
    "    theta -= eta * gradient\n",
    "    theta_history.append(theta)\n",
    "    \n",
    "    if abs(J(theta) - J(last_theta)) < epsilon:\n",
    "        break\n",
    "\n",
    "print(theta)\n",
    "print(J(theta))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xd81tXd//HXJ4MsMggJK5MRNrLCRgQRRRxo3Ru1onWP3r2t/bXV1g5re1dtHUXBURBRcRYEBYkMWWEJGGY2ZBISsuf5/ZHQImWE5ErONT7Px4MHSbiS79sIbw7nnO/3iDEGpZRSrs/LdgCllFKOoYWulFJuQgtdKaXchBa6Ukq5CS10pZRyE1roSinlJrTQlVLKTWihK6WUm9BCV0opN+HTnheLiIgw8fHx7XlJpZRyeVu2bCk0xkSe7XXtWujx8fEkJye35yWVUsrliUhGc16nUy5KKeUmtNCVUspNaKErpZSb0EJXSik3oYWulFJuQgtdKaXchBa6Ukq5CZco9CXf5bBgY7O2YSqllFM5VFzJc8v2kF9a1ebXcolCX7ozhz8v30tVbb3tKEopdU4Wbc7itW8OUlPX0ObXcolCv3lMLEcralm+O9d2FKWUara6+gYWbc7kgr6RRHcKbPPruUShj+vVmbjOgby7MdN2FKWUarZVewvIO1bNTaNj2+V6LlHoXl7CjaNi2ZhWxMGCMttxlFKqWRZuyqRLsB8X9u/SLtdziUIHuHZkND5ewnubdJSulHJ+h4orSdqbzw2jYvD1bp+qdZlCjwz24+JBXflwSzbVdbo4qpRybu9vzsIA1yfGtNs1XabQAW4eHcfRilqW7dLFUaWU82pcDM1iUkIkMeFtvxh6nEsV+vjenYkND2ShTrsopZxY0t4Cco9Vtdti6HEuVeheXsKNo2PYkFpEqi6OKqWc1PHF0KkD2mcx9DiXKnQ4YXF0c5btKEop9V8OF1eyam8+1ye232LocS5X6F2C/Zk2UBdHlVLO6f3kxsXQG0a132LocS5X6NB452hReQ3Ld+fZjqKUUv92fDH0/HZeDD3OJQt9Qu8IYsIDWKh3jiqlnMg3+wrIKani5nZeDD3OJQv9+J2j61OP6OKoUsppLNyUSaSFxdDjXLLQAa5LbFwcXaSLo0opJ5BTUsnXe/K5PjG63RdDj3PZQu8S7M9FA7rygS6OKqWcwKKmO0NvHGVnugWaUegiMk9E8kVk1wkfCxeRr0Rkf9PPndo25qkdXxzVO0eVUjYdXwyd2CfCymLocc0Zob8FTD/pY08CK40xCcDKpvfb3cQ+EcR1DmT+Bj3NSCllz4qUfHJKqrhtbJzVHGctdGPMaqDopA/PBN5uevtt4CoH52oWLy/h1jFxbE4/SkrOMRsRlFKKf25Ip0eof7s9Jvd0WjqH3tUYkwPQ9LO1/4rrEqPx8/HinfU6SldKtb8D+WWsO3CEW8bG4WNpMfS4Nr+6iMwWkWQRSS4oKHD41w8L7MDMYT34ZNshSiprHf71lVLqTOZvyKCDt5eVO0NP1tJCzxOR7gBNP+ef7oXGmDnGmERjTGJkZGQLL3dmt4+Lp7K2nsVbstvk6yul1KmUV9exeEs2M4Z0I6Kjn+04LS70z4A7mt6+A/jUMXFaZnBUKMNjw5i/IYOGBmMzilLKg3y87RCl1XXcNi7edhSgedsWFwLrgX4iki0idwN/BKaJyH5gWtP7Vt0+Lo7UwnLWHSy0HUUp5QGMMfxzfQaDeoQwIjbMdhygebtcbjLGdDfG+Bpjoo0xc40xR4wxU40xCU0/n7wLpt3NGNKdzkEddHFUKdUuNqUVsTevlNvHxSEituMALnyn6Mn8fLy5YVQMK1PyOFRcaTuOUsrNvbMhg9AAX64cGmU7yr+5TaED3NK0qX+B3miklGpD+ceqWL4rl+tGRhPQwdt2nH9zq0KPCgtg6oCuLNqcpc93UUq1mXc3ZVLXYLjV8p2hJ3OrQofGxdEj5TUs3ZljO4pSyg3V1jfw7sZMLugbSXxEkO04P+B2hT6hdwS9IoL4py6OKqXawFff55FfWs3t45xrdA5uWOheXsItY+PYmlnMrkMltuMopdzMO+vTiQoLYHI/u89tORW3K3SAa0dGE+DrraN0pZRD7csrZUNqEbeOjcPbyzm2Kp7ILQs9NMCXq4ZH8cn2QxSV19iOo5RyE299m04HH+d4bsupuGWhA9w1IZ7qugYWbtKDpJVSrXe0vIaPtmbzo+FRhAd1sB3nlNy20BO6BnN+QgTvrE+npq7BdhyllItbuDmTqtoG7pzQ03aU03LbQge4a2JP8o5V88Uu3cKolGq52voG3vk2g4l9IujXLdh2nNNy60K/ICGSXpFBzF2bhjH6FEalVMt8sSuX3GNV3DUx3naUM3LrQvfyEu6c0JPvskvYmnnUdhyllIuatzaNnhFBTO7rfFsVT+TWhQ5wzYgoQvx9mLc23XYUpZQL2pp5lO1Zxdw5IR4vJ9yqeCK3L/TADj7cNCaWL3blkH20wnYcpZSLmbc2jWB/H64ZEW07ylm5faFD4xF1IqI3Gimlzsnh4kq+2JXLTaNjCfLzsR3nrDyi0KPCApg+uBsLN2VSXl1nO45SykW8sz4DY4xTPrflVDyi0AHumtCTY1V1fLRVD5JWSp1dRU0dCzdlMn1wN6I7BdqO0yweU+gjYsMYGhPGm+vS9SBppdRZfbT1ECWVtdzlxDcSncxjCl1EuGtCPKmF5Xyzr8B2HKWUE2toMLy5Lo3zokMZGdfJdpxm85hCh8aDpLuG+DFvXZrtKEopJ7Z6fwEHC8q5a0JPpzkAujk8qtB9vb24fVw8a/YXsje31HYcpZSTmrcunS7BfswY0t12lHPiUYUOcPPoWAJ8vXljTartKEopJ5SSc4zV+wq4Y3w8HXxcqyJblVZEHhOR3SKyS0QWioi/o4K1lU5BHbhhVAyfbD9EbkmV7ThKKSfz+upUAjt4c+sY19iqeKIWF7qIRAEPA4nGmMGAN3Cjo4K1pbsn9qS+adFDKaWOO1xcyWc7DnPjqFhCA31txzlnrf33hA8QICI+QCBwuPWR2l5MeCCXndeDdzdmcqyq1nYcpZSTmLc2DQPcfb7rbFU8UYsL3RhzCPgzkAnkACXGmC8dFayt3TupF6XVdSzcqCcaKaWgpKKWhZsyuXJoD6LCAmzHaZHWTLl0AmYCPYEeQJCI3HqK180WkWQRSS4ocJ7934OjQpnQpzPz1qVRXVdvO45SyrL5GzMor6nnnvN72Y7SYq2ZcrkISDPGFBhjaoGPgPEnv8gYM8cYk2iMSYyMjGzF5Rzv3km9yTtWzafbXWKmSCnVRqpq63lzXTqT+kYysEeI7Tgt1ppCzwTGikigNO68nwqkOCZW+zg/IYIB3UN4fXWqPg5AKQ/2ybZDFJZVc98k1x2dQ+vm0DcCHwJbgZ1NX2uOg3K1CxHh3km92J9fxqq9+bbjKKUsaGgwzFmdyuCoEMb17mw7Tqu0apeLMebXxpj+xpjBxpjbjDHVjgrWXi47rztRYQH84xu90UgpT/RVSh6pheXcO6m3S93mfyqudRtUG/D19uLuiT3ZlF6k544q5YHmrE4lJjyASwd3sx2l1Ty+0AFuGBVDaIAvc3SUrpRHSU4vYkvGUX48sRc+3q5fh67/X+AAQX4+3DY2juXf55JaUGY7jlKqnbz2TSqdAn25LtH5zwttDi30JneMj8fX24vX1+jjAJTyBAfyy1iRksft4+IJ7OD854U2hxZ6k8hgP64dGc3irdnkHdOHdinl7l775iD+vl4uc15oc2ihn+C+Sb2pbzC8vlrn0pVyZ1lFFXy87RA3j46jc0c/23EcRgv9BLGdA5k5rAcLNmZypMzldmAqpZrptW8O4i3CbBe/kehkWugnuX9yH6rq6vWYOqXcVG5JFR8kZ3NdYjTdQp3+CIdzooV+kj5dOjJjSHfe/jaDkgp9tK5S7mbO6lTqjeG+C3rbjuJwWuin8MDkPpRV1/H2+nTbUZRSDlRYVs27mzK4algUMeGBtuM4nBb6KQzsEcJFA7owb10aZdV1tuMopRxk7to0qusauH+K+43OQQv9tB6Y0ofiiloWbMiwHUUp5QDFFTW88206lw3pTu/IjrbjtAkt9NMYHtuJ8xMieH1NKlW1egCGUq7urW/TKa+p54EpfWxHaTNa6Gfw4JQ+FJbV8N4mPaZOKVdWWlXLm+vSmTawKwO6u+4BFmejhX4GY3p1ZnR8OP9YnarH1CnlwuZvyKSkspYH3Xh0DlroZ/XghX3IKanio62HbEdRSrVAZU09b6xJZVLfSIbGhNmO06a00M/i/IQIhkaH8krSAerqG2zHUUqdo4WbMjlSXsNDF7r36By00M9KRHjwwgSyiir5RA+TVsqlVNXW84/VBxnTM5xR8eG247Q5LfRmuGhAFwb1COGllfup1VG6Ui7j3Y2Z5B2r5pGpCbajtAst9GYQER67qC+ZRRV8tDXbdhylVDNU1tTzStJBxvYKZ3yfCNtx2oUWejNNHdCFoTFhvLTyADV1OkpXytn9c0M6hWXVPHFxP9tR2o0WejOJCI9P68uh4koWJWfZjqOUOoOy6jpe+yaV8xMiPGLu/Dgt9HMwKSGCxLhOvPz1Ab17VCkn9va36RSV13jU6BxaWegiEiYiH4rIHhFJEZFxjgrmjESExy/uS+6xKt7dqHePKuWMjlXVMmd1KlP7d2GYm+87P1lrR+gvAsuMMf2BoUBK6yM5t/G9IxjXqzOvJB2kskZH6Uo5m7lr0iiprOWxaX1tR2l3LS50EQkBJgFzAYwxNcaYYkcFc2ZPXNyXwrJq3lmfbjuKUuoExRU1zFubxvRB3RgcFWo7TrtrzQi9F1AAvCki20TkDREJclAup5YYH86kvpG89s1BfV66Uk5kzupUymrqPHJ0Dq0rdB9gBPCqMWY4UA48efKLRGS2iCSLSHJBQUErLudcHp/Wl6MVtbylZ48q5RSOlFXz1rfpXH5eD/p1C7Ydx4rWFHo2kG2M2dj0/oc0FvwPGGPmGGMSjTGJkZGRrbiccxkWE8ZFA7owZ3UqJZV69qhStr32zUGqaut59CLPuCv0VFpc6MaYXCBLRI7vC5oKfO+QVC7i0Yv6cqyqjrlrdZSulE35x6p4Z30GVw2PctvTiJqjtbtcHgIWiMh3wDDg962P5DoGR4Vy6eBuzFubxpGyattxlPJYf191gLoGw8MXeu7oHFpZ6MaY7U3TKecZY64yxhx1VDBX8cTFfamoqePlVQdtR1HKI2UcKefdjZncMCqG+AiP2JdxWnqnaCv16RLMdSNjmL8hg6yiCttxlPI4f/lyH77eXjzqIU9UPBMtdAd4dFoCIvDXr/bZjqKUR9l1qITPdhzmronxdAnxtx3HOi10B+geGsCsCfF8vP0QKTnHbMdRymM8t2wPYYG+3HtBb9tRnIIWuoPcf0Efgv18eH75XttRlPII3x4oZM3+Qh6Y3IcQf1/bcZyCFrqDhAb68pPJffh6Tz4br7gVJk+2HUkpt2WM4blle+gR6s9t4+Jsx3EaWugONGt8PF1D/Phj7AUY22GUcmNf7MplR3YJj03ri7+vt+04TkML3YECLp7KY9s+ZVtwD5bl1DaO0nWkrpRD1dQ18PzyvSR06ciPRkTbjuNUtNAd7NqCXSQUZvDcBbOoEf32KuVoCzZmkFZYzs9n9MfbS2zHcSraOI6UlIRP0iqeKtxMengP/vnbuZCUZDuVUm6jpKKWF1fuZ0Kfzkzp18V2HKejhd4GJhenMbE4nZdW7qe4osZ2HKXcxt9X7aekspZfzBiIiI7OT6aF3gYkKYmnfnUbx6pq+fvXB2zHUcotZB6p4O1vM7h2RDQDe4TYjuOUtNDbyMAeIVw3Mpq316eTcaTcdhylXN5zy/bg7SX89BLPOvj5XGiht6EnLu6Hj5cXzy3bYzuKUi5tS0YRS3bmMHtSL7rqLf6npYXehrqG+HPvBb1YujOX5PQi23GUcknGGJ5dkkJksB+zJ/WyHcepaaG3scYRhR+/+df3NDTo7UZKnavPdhxmW2YxP724L0F+PrbjODUt9DYW2MGH/53en++yS/hwa7btOEq5lIqaOv6wdA+Do0K4dmSM7ThOTwu9HVw1LIrhsWH8adleSqv0/FGlmuuVVQfJPVbF01cM0puImkELvR14eQlPXzGIwrJq3caoVDNlFVUwZ00qM4f1IDE+3HYcl6CF3k6GxoRx3cho5q1LI7WgzHYcpZze75ak4C3Ck5f2tx3FZWiht6P/md4PPx9vnl2SYjuKUk7t2wOFLNudywNTetM9NMB2HJehhd6OugT78/DUxmemr9qbbzuOUk6prr6BZz7/npjwAH58vm5TPBda6O1s1vie9IoI4reff091Xb3tOEo5nfkbMtibV8ovZgzUZ52fIy30dtbBx4tfXTGQ1MJy3liTZjuOUk4lv7SKv3y5j/MTIrhkUFfbcVxOqwtdRLxFZJuI/MsRgTzB5H5duHRwN/729X6yiipsx1HKafxh6R6q6xp45spB+jTFFnDECP0RQFf5ztEvLx+IlwjPfP697ShKOYX1B4/w8bZD3HtBL3pFdrQdxyW1qtBFJBq4DHjDMXE8R4+wAB6ZmsCKlDxWfJ9nO45SVtXUNfCrT3cREx7AA1P62I7jslo7Qn8B+BnQcLoXiMhsEUkWkeSCgoJWXs693DWxJwldOvL057uprNEFUuW55q1LY39+GU9fMUgXQluhxYUuIpcD+caYLWd6nTFmjjEm0RiTGBkZ2dLLuSVfby9+e9Vgso9W8kqS3kGqPNPh4kpeXLGfaQO7MnWALoS2RmtG6BOAK0UkHXgPuFBE5jsklQcZ26szVw+P4rVvDnJQ7yBVHuiZz3djMPz6ioG2o7i8Fhe6MebnxphoY0w8cCPwtTHmVocl8yBPzRhAgK83T320Ux+xqzzKl7tzWb47j4cuTCC6U6DtOC5P96E7gchgP56aMYCNaUW8n5xlO45S7aK0qpZffbqb/t2C9eAKB3FIoRtjkowxlzvia3mqG0bFMKZnOL9fmkJ+aZXtOEq1uT8t20teaRV/vOY8fL11bOkI+l10EiLCH340hKq6Bp75TPemK/e2JaOI+RszmDU+nmExYbbjuA0tdCfSK7IjD1/YhyU7c3RvunJb1XX1PLl4J91D/Hni4n6247gVLXQnM3tSb/p1DeaXn+7S042UW3otKZX9+WU8e/VgOuoZoQ6lhe5kOvh48cdrhpB7rIrnl++1HUcphzqQX8rLqw5w+XndubC/7jl3NC10JzQ8thOzxsfzzvoM1h88YjuOUg5RV9/AEx98R5CfN7++YpDtOG5JC91J/eyS/sR1DuRni3dQXl1nO45Srfb6mjR2ZBXzzMzBRAb72Y7jlrTQnVRAB2+ev3Yo2Ucr+dOyPbbjKNUq+/NK+etX+5g+qBtXnNfddhy3pYXuxEb3DGfW+HjeXp/B+itvg8mTbUdS6pzV1Tfw0w8bp1p+e9Vgfc55G9JCd3L/nnrpNZ1yL1/bcZQ6ZzrV0n600J1cwMVTeX71G2T7hfCcd5/GUbqO1JWL0KmW9qWF7gJGlx5i1pbPeWfk5awJjbMdR6lmqalr4LH3t+tUSzvSQnd2SUmQlMT/1qfSp6KQn465jeKlX9pOpdRZvbhyH7sOHeMPPzpPp1raiRa6i/A3dbxwYAlF5TU89fFOjNHH7CrntTm9iFeTDnJ9YjTTB3ezHcdjaKG7iqQkBi99n8en9WPpzlw+2nrIdiKlTqm0qpbHFm0nulMgv9IbiNqVFrqLmT2pF6Pjw/n1Z7vJKqqwHUep//LM599zuLiSv94wTJ/V0s600F2Mt5fwl+uHIsDj72+nXk84Uk7ki505fLglmwen9GFkXCfbcTyOFroLigkP5DdXDWJz+lFeWaWHSyvncLi4kp9/vJOh0aE8NDXBdhyPpIXuoq4aFsVVw3rw1xX72JRWZDuO8nB19Q08vHAbdfWGF28cricQWaLfdRclIjx79RDiOgfx8MJtFJXX2I6kPNgLK/aTnHGU3109mPiIINtxPJYWugvr6OfD324aTlF5Df/zwQ7dyqisWLu/kJeTDnBDYgwzh0XZjuPRtNBd3OCoUJ6a0Z+Ve/KZuzbNdhzlYQpKq3l00Xb6RHbk6St1i6JtWuhu4I7x8Vw8sCvPLdvDjqxi23GUh2hoMDz+/nbKqmt5+ZYRBHTwth3J42mhuwER4flrh9Il2J/7F2zlqM6nq3bwt68PsGZ/IU9fMYi+XYNtx1G0otBFJEZEVolIiojsFpFHHBlMnZvQQF9evmUEBaXVPLJI96ertrVqbz4vrNzHNSOiuWFUjO04qklrRuh1wBPGmAHAWOABERnomFiqJYbFhPH0lYNYva+AF1futx1HuamsogoefW87A7qF8Lur9SmKzqTFhW6MyTHGbG16uxRIAXSJ27KbRsdw3choXlq5n5UpebbjKDdTVVvPffO3YIzhtVtH4u+r8+bOxCFz6CISDwwHNp7i12aLSLKIJBcUFDjicuoMRITfXjWYQT1CeGzRdjKOlNuOpNyEMYb/98kudh8+xos3Die2c6DtSOokrS50EekILAYeNcYcO/nXjTFzjDGJxpjEyMjI1l5ONYO/rzev3ToSEeG++VupqKmzHUm5gQUbM/lwSzaPTE1gSv8utuOoU2hVoYuIL41lvsAY85FjIilHiAkP5MUbh7E39xhPvL+DBl0kVa2w/uARnv5sN1P6RfKIPqfFabVml4sAc4EUY8z/OS6ScpTJ/brw1IwBfLErlxd0kVS1UMaRcn6yYAvxEUG8eNNwvLx0EdRZtWaEPgG4DbhQRLY3/ZjhoFzKQe6e2PPfi6Sf7zish0yrc1JaVcvdbycD8MbtiYT4+1pOpM6kxU+fN8asBfSvaifX+BCvwaQfKeenH+wgLqgr55Xr7hd1dvUNhocXbiO9sJx37h6tD91yAXqnqAfw8/Hm1Q+fJaL8KPfEX0belp06Uldn9YelKazaW8AzMwcxvneE7TiqGbTQPUREXQVv7P2Ysg4BzLr2aUq9O9iOpJzYO+vTeWNtGneMi+OWMXG246hm0kL3FElJDFj6AS+nLmFfl3juv/m31K782nYq5YSW787l15/t5qIBXfjl5XrztyvRQvcwk0vS+UPqctbsL+TJxTv1GerqB7ZmHuXhhds4LzqMl24ajo+ePORS9EhuT5OUxPXA4RX7eGHFfqI6BfD4tL62UyknkFZYzo/fTqZbqD9z70gksIPWg6vR/2Me6pGpCRwuruSllfvpHurPTaNjbUdSFhWWVTPrzU0AvH3naCI6+llOpFpCC91DiQi/u3oI+aXV/OLjnYT4+3LZed1tx1IWlFTUctvcTeQdq+Lde8bq9kQXphNkHszX24tXbhnByLhOPLpoG6v25NuOpNpZeXUdd761iYP5Zcy5LZERsZ1sR1KtoIXu4QI7+DB31ij6dQvmvvlb2JB6xHYk1U6qauuZ/c9ktmcV89JNw5jUVx+e5+q00BUh/r68fedoYsID+fHbyXouqQeorW/goYXbWHfgCM9fO5Tpg3W6zR1ooSsAOnf0Y/7dY+gU5Msdb25i9+ES25FUG6mrb+Dx93fw1fd5/GbmIK4ZGW07knIQLXT1b91C/Vlw91gCfb25+fWN7Dqkpe5uausbeGTRdj7fcZgnL+3P7ePibUdSDqSFrn4gtnMgi+4dR0c/H25+fYNOv7iR2voGHl64jSXf5fCLGQO474LetiMpB9NCV/8lJjyQ92aPJTTQl1vnbmRb5lF9mJeLq6lr4MF3t/LFrlx+eflA7pnUy3Yk1Qa00NUpNZb6ODoFduC2uZtI7qjnf7uqqtp67l+wleW783j6ioHcPbGn7UiqjWihq9OKCgtg0Yq/0KU4n1v7/Yivs8p1pO5ijlXVcse8TaxIyeO3Vw1m1gQtc3emha7OqHtNGe/vXkhCYRb3XPNLFkcMsh1JNVN+aRU3/GMDWzKO8uKNw7htrD4G193prf/qzJKSiAAWXjiNewOv5ok+Mzj6yBP82HYudUYZR8q5be4mCkqrmTtrFBfoTUMeQUfoqlk6NtQyb89iZgzpxrNLUvj90hQaGvTRu85o16ESrnl1Pceqann3njFa5h5ER+iqeZKS8AP+1mDoHLSbOatTyThSzl9vGKaPWXUiy3bl8Oii7XQO8uO9u8bQp0uw7UiqHekIXZ0Tby/hNzMH8avLB/LV93lc99p6ckoqbcfyeMYYXl51gPvmb2VA9xA+eWCClrkH0kJX50xEuGtiT+bOGkXGkQpm/n2d3oBkUXVdPU98sIPnl+/lyqE9WHjPWCKD9XnmnqhVhS4i00Vkr4gcEJEnHRVKuYYp/bqw+Cfj8fX24vp/rOfDLdmNv6BbG9vN4eJKbpyzgY+2HuLxaX158cZh+Pt6246lLGlxoYuIN/AycCkwELhJRPREWQ/Tr1swnz44gRGxnfjpBzt4cvF3VIkWSntYs7+Ay/+2ln25pbxyywgenpqAiNiOpSxqzQh9NHDAGJNqjKkB3gNmOiaWciURHf34592juf/QBt7bnMU1cVeSuX2PjtTbSEOD4cUV+7l93iYiOnbgs4cmMmOIPv5Wta7Qo4CsE97PbvqY8kA+3l78LGsNc/csJiusG5fNepEvwvXwaUfLL61i1lub+euKfVw9LIpPHphA78iOtmMpJ9Ga/Wan+rfdf21MFpHZwGyA2Fg9iNitJSUxFVhyyUweSLiCn/SdyY9GRPF0VS0h/r6207m8Zbty+PlHO6moqed3Vw/m5tGxOsWifqA1I/RsIOaE96OBwye/yBgzxxiTaIxJjIzUGxw8QUx1CYt3v8vDUxP4dPthLn1hjR5t1wrHqmp54v0d3Dd/K9GdAlny8ERuGROnZa7+ixjTsrv9RMQH2AdMBQ4Bm4GbjTG7T/c5iYmJJjk5uUXXU65pa+ZRHl+0nYyiCu6a0JPHp/UlyM/nP3PrSUk24zm9tfsL+d/F35FTUsmDU/rw0NQEfL11t7GnEZEtxpjEs72uxVMuxpg6EXkQWA54A/POVObKM42I7cTSR87nd0tSmLs2jWW7cnnmykFcZDvl/p5oAAAIR0lEQVSYk8svreLZf6Xw2Y7D9IwI4oP7xjMyrpPtWMrJtXiE3hI6Qvdsm9OL+MX/fca+wEgu2fstv145hx4jmp7eqCN1oHEHy8LNmfzxiz1U1zZw3+Te3D+5t+4t93BtPkJX6lyNig/nXzvf4Y3uibzUawwX9XyV+/K28OOcZAJth3MCG1OP8PulKezILmFsr3B+d/UQ3cGizomO0JUVWZfM5Nm4ySwP70uXYD8en9aXa0dG4+Pt5XHz6wfyS/njF3tYkZJPtxB/fja9H1cPj9JFT/VvOkJXTi2muoR/7PuU5Lc+4vdLU3jyo53MXZvG49P6cgme8ZChrKIKXkk6wKLNWQR18OFn0/tx14SeOr2iWkxH6Mo6YwzLd+fyp9dXkBoQTu8jWfxkwwfMjDD4mga3G6nvyyvl1aSDfLbjMF4Ct4yJ46EL+9C5oz5QS52ajtCVyxARpg/uzrQd81jauS8vhw7hp5c9zl+rS7g7J5lrKmoJnTGt8cUuWu7GGDamFTFvbRpffp9HgK83d46P58fn96JbqL/teMpN6AhdOR0zeTJJYT15+ZJ7SM44ir+vF5cd/o6b83Yw4vMF/5lbdoG59qPlNSzems27mzJJLSgnNMCXO8bHc+f4eDoFdbAdT7kIHaErlyXAlOI0pvxkPLtmXM+CrsP4LCyBxZGD6f/w21xxJIXLXvoV8Sd/opMUfGVNPUl781myM4cvv8+jpq6BEbFh/Pm6oVw2pDsBHXSOXLUNHaEr59ZU0mXrN/HZgEm8P+Eatgf3AGBAXioz9q7jwrAGBlTk/2chNSnp9E95bKOyLyqvYd2BQpbtyuXrPflU1tYTHtSBy8/rzk2jYxnQPaRNrqs8Q3NH6FroyjWcMPo+dPEVLAvvyxc+XUmObrwxKayqlLEZ3zEucyejuvqTsOFrfM8b8sOvsX37D98vK4P6+v+838w/C8YYCsqq2ZFVwrcHC1l/8Ah7ckuBxkcJTx/clRlDujM6PrxxG6ZSraSFrtzLqaZTJk8mzzeIb0Pj+DYklm/9unIotAsAvvW19CnMYkB+GgMK0okuyaNbaSHdK44SWV6Md1DgWQu9tKqW3JIqDpdUkVtSyYH8MlJySknJOcaR8hoA/Hy8GBUfzrjenRnbqzPDYsLw9tL948qxtNCV+zup5M3kyWQdPMTW0ReRcqSKPV3iSYnsSX5w5x98mndDPR2rKwioq8a/tgb/umq8G+qp8vWjyqcD1VExlFfXU1lb/4PP6+DjRb+uwfTvFsyA7iEM6hHCsNgw/Hx0Tly1LV0UVe7vpPlwAWJ7RxG7+FWumjwZ8tMgfxXFKfs57BVAbmAncoIjyAmJoNQviCqfDlT6+lHl40e9lzf+ddX41dYQcPEoAny96RLiR7fQALqH+tMtxJ/uof46haKcmha6ch+nWfAMG5BAGDBw+3Y4sv8/v9DCOXSlnJUWunJPpyr3yZP/e2FUKTeiha48hxPfgKSUI+iEoFJKuQktdKWUchNa6Eop5Sa00JVSyk1ooSullJvQQldKKTehha6UUm5CC10ppdyEFrpSSrmJdn3aoogUABkt/PQIoNCBcdqaK+XVrG3HlfK6UlZwrbytzRpnjIk824vatdBbQ0SSm/P4SGfhSnk1a9txpbyulBVcK297ZdUpF6WUchNa6Eop5SZcqdDn2A5wjlwpr2ZtO66U15WygmvlbZesLjOHrpRS6sxcaYSulFLqDFyq0EXkOhHZLSINIuKUq9siMl1E9orIARF50naeMxGReSKSLyK7bGc5GxGJEZFVIpLS9HvgEduZzkRE/EVkk4jsaMr7jO1MZyMi3iKyTUT+ZTvL2YhIuojsFJHtIuLUJ8+LSJiIfCgie5p+/45rq2u5VKEDu4AfAattBzkVEfEGXgYuBQYCN4nIQLupzugtYLrtEM1UBzxhjBkAjAUecPLvbTVwoTFmKDAMmC4iYy1nOptHgBTbIc7BFGPMMBfYuvgisMwY0x8YSht+j12q0I0xKcaYvbZznMFo4IAxJtUYUwO8B8y0nOm0jDGrgSLbOZrDGJNjjNna9HYpjX8oouymOj3TqKzpXd+mH067YCUi0cBlwBu2s7gTEQkBJgFzAYwxNcaY4ra6nksVuguIArJOeD8bJy4dVyUi8cBwYKPdJGfWNIWxHcgHvjLGOHPeF4CfAQ22gzSTAb4UkS0iMtt2mDPoBRQAbzZNZ70hIkFtdTGnK3QRWSEiu07xw2lHuieQU3zMaUdlrkhEOgKLgUeNMcds5zkTY0y9MWYYEA2MFpHBtjOdiohcDuQbY7bYznIOJhhjRtA4vfmAiEyyHeg0fIARwKvGmOFAOdBma2s+bfWFW8oYc5HtDK2QDcSc8H40cNhSFrcjIr40lvkCY8xHtvM0lzGmWESSaFyvcMYF6AnAlSIyA/AHQkRkvjHmVsu5TssYc7jp53wR+ZjG6U5nXFvLBrJP+NfZh7RhoTvdCN3FbQYSRKSniHQAbgQ+s5zJLYiI0DgPmWKM+T/bec5GRCJFJKzp7QDgImCP3VSnZoz5uTEm2hgTT+Pv2a+ducxFJEhEgo+/DVyMc/5FiTEmF8gSkX5NH5oKfN9W13OpQheRq0UkGxgHLBGR5bYzncgYUwc8CCyncdHufWPMbrupTk9EFgLrgX4iki0id9vOdAYTgNuAC5u2qm1vGlE6q+7AKhH5jsa/6L8yxjj9dkAX0RVYKyI7gE3AEmPMMsuZzuQhYEHT74VhwO/b6kJ6p6hSSrkJlxqhK6WUOj0tdKWUchNa6Eop5Sa00JVSyk1ooSullJvQQldKKTehha6UUm5CC10ppdzE/wei03m/ov3tTQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(plot_x, plot_y)\n",
    "plt.scatter(np.array(theta_history), J(np.array(theta_history)), color='r', marker='+')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(theta_history)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 修改学习率，查看梯度下降法寻找最小值的过程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(initial_theta, eta, epsilon=1e-8):\n",
    "    theta = initial_theta\n",
    "    theta_history.append(theta)\n",
    "    \n",
    "    while True:\n",
    "        gradient = dJ(theta)\n",
    "        last_theta = theta\n",
    "        \n",
    "        theta -= eta * gradient\n",
    "        theta_history.append(theta)\n",
    "        if abs(J(theta) - J(last_theta)) < epsilon:\n",
    "            break\n",
    "            \n",
    "def plot_theta_history():\n",
    "    plt.plot(plot_x, J(plot_x))\n",
    "    theta_history_np = np.array(theta_history)\n",
    "    plt.plot(theta_history_np, J(theta_history_np), color='r', marker='+')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "424\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VdW9NvDnlzmQCchAQhLCPI9GRgUULCqKrdqKU8VabWuttrX6WntvCfd9P2p7bavV1hYVpYKCUqq2DqhIBJkTZghjyEwGAmSezsl6/zgnEEMmcoa19z7P9/PJJzmbA/sxwsNi7bXXFqUUiIjI/Px0ByAiIvdgoRMRWQQLnYjIIljoREQWwUInIrIIFjoRkUWw0ImILIKFTkRkESx0IiKLCPDmyaKjo1VKSoo3T0lEZHqZmZlnlFIxXb3Pq4WekpKCjIwMb56SiMj0RCS3O+/jlAsRkUWw0ImILIKFTkRkESx0IiKLYKETEVkEC52IyCJY6EREFmGKQv9o/2ms2tGtZZhERIZSeL4Oz31yBKVV9R4/lykK/eMDp/H8+qNosNl1RyEiuixrduXj75tOotHW7PFzmaLQ77gyCedqm7D+UInuKERE3WZvVngvIx9XD4tBYp9eHj+fKQr9qqHRSOwTijW78nRHISLqtk3HynC6oh53XpnklfOZotD9/AR3pCZhy4ly5JbX6I5DRNQt7+zMQ3RYEOaOivPK+UxR6ABwe2oi/MQxH0VEZHSllfXYcKQUt01ORFCAd6rWNIUeHxmKa0bE4r3MAtjsnr+4QETkirW7C2BvVrjDS9MtgIkKHQAWTUlGWVUDvjxSqjsKEVGHmpsV1uzKx9RBfTE4Jsxr5zVVoV8zIgax4cFYzWkXIjKw7dnlyC2vxaIp3hudAyYr9AB/P3w3NRHpR0txuqJOdxwiona9sysfESEBuGFsvFfPa6pCB4A7UpPRrID3Mgp0RyEiusS5mkasP1iMWycnIiTQ36vnNl2hJ/frhauGRmPNrnw0NyvdcYiIvmHdnkI02pu9ejG0hekKHQAWTUlC4fk6bD5xRncUIqILlFJYvTMPE5KiMCo+wuvnN2WhXzc6Dn16BWL1Tt45SkTGsTvvHI6XVnvtztC2TFnowQH+uG1yIj4/XIIz1Q264xARAQBW78xHryB/3DQhQcv5TVnogGPaxdas8M9MXhwlIv2q6pvwn/2nsXBCAsKCA7RkMG2hD40NR+rAPlizKx9K8eIoEen1wd4i1DXZsWhKsrYMpi10wHHnaPaZGmzPPqs7ChH5MKUUVu/Kw8j+4ZiQGKktR5eFLiLLRaRURA62OtZXRD4XkePOz308G7N9C8bFIyIkgE8zIiKt9hVU4GBhJe6ckgwR0ZajOyP0NwFc3+bYUwA2KKWGAdjgfO11oUH+uP2KJKw/VIyyKl4cJSI9Vm7PRa8gf9w6eYDWHF0WulJqE4C2cxq3AFjh/HoFgG+7OVe33T0tGU12hXczuL8LEXnf+dpG/HtfEW6ZOADhIYFas/R0Dj1OKXUaAJyfYzt6o4g8JCIZIpJRVlbWw9N1bEhMGGYM6Ye3d+TBzjtHicjL1mYWoMHWjHum6bsY2sLjF0WVUsuUUqlKqdSYmBiPnOPeaQNReL4OG7mtLhF5UXOzwqodeZicHIUxCfouhrboaaGXiEg8ADg/a23SeaPjEBsejJW8OEpEXrT1ZDlOnanBPdMG6o4CoOeF/iGA+5xf3wfgA/fE6ZlAfz8smpKMr46VIa+8VmcUIvIhK7fnok+vQNw4zrvb5HakO8sW3wGwDcAIESkQkQcAPAfgOhE5DuA652ut7pySBD8RrNrJUToReV5xRT0+zyrB91KTvL5Nbke6vD9VKXVnBz80181ZXBIfGYp5o2LxXkYBfnndcAQHGOMbTETW9M5Ox0KMu6bqvxjawtR3irZ1z7SBOFvTiE8OFOuOQkQW1mRvxupdeZg9PAYD+/XWHecCSxX6zCHRSOnXCyu3c9qFiDxnQ1YJSiobDHMxtIWlCt3PT3DPtIHIyD2HrNOVuuMQkUW9tT0XCZEhuHZkh7fgaGGpQgeA269IRHCAH0fpROQR2WXV2HKiHHdNTYa/n759W9pjuUKP6hWEm8Yn4P09haiqb9Idh4gsZtWOPAT4Cb6n6alEnbFcoQPAvdMHoqbRjvf3FOqOQkQWUtdox3sZ+Zg/tj9iw0N0x7mEJQt9QmIkxg6IwFvbc/nwCyJym3/vK0JlvQ33TDXWxdAWlix0EcH3p6fgWEk1tp0s1x2HiCxAKYU3tuZgRFw4pg3uqztOuyxZ6ACwcEIC+vYOwhtbc3RHISIL2HnqLLJOV2LxzBStD7HojGULPSTQH3dNScYXWSXIP8v9XYjINW9uzUFUr0B8e6Leh1h0xrKFDjjuHPUTwQqO0onIBQXnarH+UDEWXZmM0CDjbiti6ULvHxmCG8b2x5qMfNQ02HTHISKTemt7LkQE90435sXQFpYudAC4f+YgVNXbsG53ge4oRGRCdY12rN6Zj/lj4jAgKlR3nE5ZvtAnJ0dhfGIk3tyag2Y+oo6ILtO/9hSioq4Ji2cM0h2lS5YvdBHB/TNTcLKsBl+fOKM7DhGZiFIKb249hTEJEbgypY/uOF2yfKEDwI3j4hEdFow3tpzSHYWITGTbyXIcK6nG4hnGXarYmk8UenCAP+6emoyNR8tw6kyN7jhEZBLLt+SgX+8g3DwhQXeUbvGJQgeAu6clI9CfSxiJqHvyymux4UgJ7pqabJhHzHXFZwo9NjwEN41PwHsZ+dyFkYi69I9tOfAXwd0G3belPT5T6ACweEYKahrtWJvJJYxE1LGaBhvWZOTjhnHx6B9pvF0VO+JThT4hKQqTk6OwgksYiagT63YXoKrehvtnpuiOcll8qtABYPHMQcgpr8WXR0p1RyEiA2puduyqOCExEpOSonTHuSw+V+g3jO2P+MgQvPZ1tu4oRGRAG4+WIrusBj+4apApliq25lKhi8gvROSQiBwUkXdExPCTTYH+fvjBzEHYnn0W+wvO645DRAazbFM2BkSF4sZx8bqjXLYeF7qIDADwKIBUpdRYAP4AFrkrmCctmpKE8OAAvLqZNxoR0UX78s9jx6mzuH9mCgL9zTeB4WriAAChIhIAoBeAItcjeV54SCDunJqMjw+c5l7pRHTBq5uzER4SgEVTknVH6ZEeF7pSqhDA8wDyAJwGUKGU+sxdwTxt8YwUCIA3tuTojkJEBpB/thYfHziNu6YkIyw4QHecHnFlyqUPgFsADAKQAKC3iNzTzvseEpEMEckoKyvreVI3S4gKxc0TErB6Vx4qanmjEZGvW77lFPxEsNhkSxVbc2XKZR6AU0qpMqVUE4B1AGa0fZNSaplSKlUplRoTE+PC6dzvh1cPQm2jHW/vzNMdhYg0qqhtwppd+Vg4IQHxkcbe87wzrhR6HoBpItJLHGt75gLIck8s7xiTEImrhkbjjS2n0Ghr1h2HiDRZtTMXtY12/PDqwbqjuMSVOfQdANYC2A3ggPPXWuamXF7z4KzBKK1qwIf7THE9l4jcrMFmx5tbcnD1sGiMTojQHcclLq1yUUotUUqNVEqNVUrdq5RqcFcwb5k1LBoj4sLx2uZsKMXtAIh8zYd7i1Ba1YAHTT46B3zwTtG2RAQPzhqMI8VV2HycTzQi8iVKKby6ORsj+4fj6mHRuuO4zOcLHQAWTkhAXEQwXt3M7QCIfMlXx8pwrKQaD1492HS3+beHhQ4gKMAPi2cMwubjZ3C4qFJ3HCLyklc3ZyMuItg0TyTqCgvd6a6pyegd5M9ROpGPOFhYgS0nynH/zEEICrBGFVrjv8INIkMDcceVyfhwXxG3AyDyAX/76iTCggNwp0lv828PC72VB2cNgp+Ao3Qii8suq8ZHB07jnmkDERkaqDuO27DQW4mPDMVtkxOxelc+SqvqdcchIg/5+1fZCPL3wwNXDdIdxa1Y6G38aPYQ2OzNWP51ju4oROQBRefrsG5PAe64Mgkx4cG647gVC72NQdG9sWB8AlZuz+WmXUQW9OrmbCgFPDTL/DcStcVCb8fDc4agusGGf2zL0R2FiNyovLoB7+zMwy0TByCxTy/dcdyOhd6OUfERmDsyFsu3nEJto013HCJykze25KDB1oyfzLHe6BxgoXfo4WuG4lxtE97Zma87ChG5QWV9E1Zsy8H1Y/pjaGy47jgewULvwBUD+2Da4L5YtukkGmx23XGIyEUrt+eiqt6Gn14zVHcUj2Ghd+Kn1wxFSWUD1u0u1B2FiFxQ12jH65tPYfbwGIwdEKk7jsew0Dtx1dBojE+MxN++OgmbnQ/AIDKrNbvyUF7TaOnROcBC75SI4OE5Q5FbXouPDpzWHYeIeqDR1oxlm7JxZUofTBnUV3ccj2Khd+Fbo+MwNDYMf914Es3NfAAGkdm8v7cQRRX1eNjio3OAhd4lPz/Bw3OG4GhJFTYcKdUdh4gug71Z4W/pJzE6PgJzhhvrIfWewELvhpsnJGBgv154ccMxPqaOyEQ+3FeI7DM1+Nm1Qy3xAIuusNC7IdDfD49cMxQHCyvx+eES3XGIqBts9mb8ecMJjOwfjvlj+uuO4xUs9G76zqQBSOnXCy98cZyjdCIT+GBvEU6dqcHP5w2Hn5/1R+cAC73bAvz98LNrh+Hw6UqsP8RROpGR2ezNeOnL4xgdH4H5Y+J0x/EaFvpluGViAgZF98YLXxzjihciA3t/bxFyymvx83nDfGLuvIVLhS4iUSKyVkSOiEiWiEx3VzAjcozSh+JIcRXWHyrWHYeI2tEyOh+TEIHrRvvO6BxwfYT+IoBPlVIjAUwAkOV6JGNbOCEBg6N744UvjnOUTmRA6/YUIre8Fj+fN9ynRueAC4UuIhEAZgF4HQCUUo1KqfPuCmZUAf5+eHTuMBwtqcInBzlKJzKSJufofNyASMwbFas7jte5MkIfDKAMwBsiskdEXhOR3m7KZWg3T0jAkJjeeHED59KJjGTd7gLkn63zubnzFq4UegCAyQBeUUpNAlAD4Km2bxKRh0QkQ0QyysrKXDidcfj7CR6dOwzHSqq5xwuRQTTamvHSlycwITES1470vdE54FqhFwAoUErtcL5eC0fBf4NSaplSKlUplRoTY51bb28an4ChsWF4ccNx2DlKJ9Lun7sLUHCuzifnzlv0uNCVUsUA8kVkhPPQXACH3ZLKBPz9BI/NHYYTpdX4z/4i3XGIfFqjrRkvf3kCE5OiMGeEdQaOl8vVVS4/A7BKRPYDmAjgGdcjmceCcfEYHucYpXO/dCJ93svMR+F53507b+FSoSul9jqnU8Yrpb6tlDrnrmBm4Ocn+MW84cguq8G6PXyqEZEO9U12/HnDcVwxsA9m+8COip3hnaIuun5sf4xPjMSLXxzns0eJNFixNQcllQ14cv4Inx6dAyx0l4kInpw/EoXn67Bqe57uOEQ+paKuCX9NP4k5I2IwdXA/3XG0Y6G7wVXDojFjSD/8ZeMJVDfYdMch8hmvbspGRV0TfvWtEV2/2Qew0N3kifkjUF7TiOVfn9IdhcgnlFU1YPmWU7hpfDzGDojUHccQWOhuMim5D741Og7LNmXjbE0jkJamOxKRpf1l4wk02JrxOEfnF7DQ3ehX80egttGGV9JPAEuX6o5DZFn5Z2uxakcuvpeahEHRPrHjSLcE6A5gJcPjwvGTfvW4+qd36o5CZGl/+uIY/MRxcx9dxBG6O6Wl4Yknbse03P2O1yKOD06/ELnN4aJK/GtPIRbPSEH/yBDdcQyFhe5OaWmAUth26/0AgOz1m4AlS1joRG707CdZiAgJxMNzhuqOYjgsdA8Y/dLvAAB1j/2Sc+lEbrTpWBk2Hz+Dn107FJG9AnXHMRwWugdEJsQgf+osjDmSoTsKkWXYmxWe+TgLSX1Dce/0gbrjGBIL3RPS0pC0Y9PF15xLJ3LZv/YU4khxFZ6cPxLBAf664xgSC90TnHPpO//4GgAg8+nnOJdO5IL6Jjv+8NlRTEiKwk3j43XHMSwWugelPuq4OJry599xLp3IBa9/fQqnK+rx9A0jfX4Drs6w0D3Iz98Ppd+7B/2qfWpXYSK3Kq9uwN/ST2LeqDhuwNUFFronpaUh9t2VF19zLp3osv3h82Ooa7LjqRtG6o5ieCx0T3LOpednHAQA7JqzkHPpRJch63QlVu/Mw73TB2JobJjuOIYnSnnvAcepqakqI8NHl/KJoBkCPyjAi99zIrNSSuGuV3cgq7gS6b+ag6heQbojaSMimUqp1K7exxG6lzQ8/itUhjpGGKqZzx8l6spnh0uwLbscv7xuuE+X+eVgoXtDWhqC//A8ouqqAADi78+5dKJONNjseObjLAyLDcNdU5J1xzENFro3OOfS7Y1NAIDiyNiLx4noEm9syUFueS3++6bRCPBnTXUXv1Ne5B/o2K24f0Wp5iRExlVW1YCXvzyBuSNjMWt4jO44psJC96a2I3IuYyS6xO8/PYL6Jjt+s2CU7iimw0L3JufUS9GB4wCA/ZNncxkjUSuZuefwXmYBHrhqEAbHcJni5XJ52aKI+APIAFColLqps/f69LLFtlrfvsxljESwNyssfPlrnKluwIbH5yAsmA9Ua+HNZYuPAchyw6/jU2y/fhp50YkAgMbaes1piPR7e0cuDhVV4r8WjGaZ95BLhS4iiQAWAHjNPXF8RFoaAp59BslnCgAAQb1DOZdOPq28ugH/u/4oZgzpx90UXeDqCP0FAE8C6PBOGRF5SEQyRCSjrKzMxdNZhHMuvWWqpSYo9OJxIh/0+0+PorbRjqULx3A3RRf0uNBF5CYApUqpzM7ep5RappRKVUqlxsRwCVJ7Auw23RGItNmddw5rMvJx/8wUDIsL1x3H1FwZoc8EsFBEcgCsBnCtiKzs/KfQNzhH5MF2xw1HXMZIvsZmb8ZvPziIuIhgPDZvuO44ptfjQldK/VoplaiUSgGwCMCXSql73JbMFzinXuqragAAhX0TLh4n8gErtuXiYCEvhLoL16EbQEhYLwDAgLNFmpMQeU/R+Tr84bOjmDMihhdC3cQtha6USu9qDTp1gneQkg9a8uEhNCuF/3vLWF4IdROO0I3AOfVSnlMIADg6eOzF40QW9OnBYnx+uAS/mDccSX176Y5jGSx0A+k30DGHPiL7oOYkRJ5TVd+EtA8PYWT/cPzgqkG641gKC91IOPVCPuAPnx1DSVU9nr11HAK5Na5b8btpJM6pl9xtuwEAu6ddd/E4kQXsyTuHFdtycO+0gZiU3Ed3HMthoRvQwGmTAACTt3+uOQmR+zTY7Hhy7X70jwjBE/NH6I5jSSx0I+LUC1nQSxtO4HhpNZ65dRzCQwJ1x7EkFroROadejn+UfvGYUix0Mq2DhRV45auTuG1yIq4ZEas7jmWx0A1s2I2zL3x9YNnbLHQypSZ7M55Yux99ewfhv2/iU4g8iffaGlVaGrB06YWX43509zd/jMgkXkk/iazTlfj7vVcgqleQ7jiWxhG6UbXZYtcmfhePE5nE0eIqvPTlcdw8IQHzx/TXHcfyWOhG5yzwAOXccp4XSMkkGm3NePy9vQgPCUTazaN1x/EJLHSjS0sDliy5sCNjaXi/i8eJDOylL4/jYGElnvnOOPQLC9Ydxyew0M0gLe3Cjox9q8994ziREe3OO4e/bDyB2yYn4vqxnGrxFl4UNYv2pl7a/BiREdQ22vD4u/sQHxmKJQs51eJNHKGbhfMiqa2uHgBQERJ28TiRgTz78RGcOlOD5787ARG8gcirWOgmE/DcswCAyPpqxwFeJCUD+epYGd7anosHrhqE6UP66Y7jczjlYjbO4t7aGIoZzz7lOOZc2kik09maRjzx3j4Miw3jXi2acIRuRmlpmP7/nrjwMvvzrzlCJ62UUnjivX04X9uEFxZNREigv+5IPomFbkZpaRD/i39gBn/rasddpSx10uTNrTnYcKQUT984EmMSInXH8VksdDNqcxfpN46z1MnLDhVV4NmPj2DeqFjcNyNFdxyfxjl0M2tvm92OfozIA2obbfjZO3vQp3cgfn/7BD7sWTOO0M3MeRdpk3MpY3UQH7ZL3pX24SGcOlODP90xEX17c+Mt3VjoZpeWhkDnUsawxlrHsaVLuZSRPO79PYV4N6MAj1wzFDOGROuOQ3Ch0EUkSUQ2ikiWiBwSkcfcGYwug3OknvnnNy8e41JG8qBjJVX49boDmDKoLx6bO0x3HHJyZQ7dBuBxpdRuEQkHkCkinyulDrspG12mKx5dfPEF59PJQ6obbPjxykz0Dg7Ay3dOQoA//6FvFD3+P6GUOq2U2u38ugpAFoAB7gpGl8m58sX+m98AAKqDQvXmIUtSSuH/rN2P3PJavHzXJMRGhOiORK245a9WEUkBMAnADnf8etRz/gGOf3SFNdY5DnA+ndxo+ZYcfHTgNJ6YPwLTBvPWfqNxudBFJAzAPwH8XClV2c6PPyQiGSKSUVZW5urpqCvO+fRDK/554ZCy2/XlIcvIzD2LZz/OwnWj4/CjWYN1x6F2uLQOXUQC4SjzVUqpde29Rym1DMAyAEhNTeWVOi8Zc99tF75ufVcpR+rUE8UV9fjxyt0Y0CcUz3+X682NypVVLgLgdQBZSqk/ui8Sucw5n65++1sAQDP4h496rr7JjofeykBtgw2vfj8VkaHcEteoXJlymQngXgDXishe58eNbspFbtAyivKD8x9GLfPpc+boC0WmopTCU//cjwOFFXhh0SQMjwvXHYk60eMpF6XU1wCHfobmnF45/e3vIX7SGABAZXEZIvrHaAxFZrJsUzbe31uEX31rOK4bHac7DnWBC0h9QEuZA7hY5hylUxc2Hi3Fc58ewYLx8fjpNUN1x6FuYKFbXcvOjLNnf/P4V19xOSN16GhxFR59ew9G9Y/A/94+nhdBTYKF7ivS0wEAW7+z+OKxJUtY6HSJksp63P/GToQG+eO1+1LRK4ibspoFC92XzJ6NGf968+JrXiSlNmoabPjBm7tQUdeE5YuvREIU7zg2Exa6L0lPB5RC05NPXTh07KN0x/QL+TybvRmPvL0bR4qr8PLdkzF2AJ88ZDYsdF+TlobA3z934eXwBXMcX3CU7tOUUljy4SFsPFqG/7llDK4ZEas7EvUAC93X8CIpteMvG09g1Y48/Hj2ENw9daDuONRDLHRf5bxIenjdp3pzkHZvbc/F858dw3cmDcCT80fojkMuYKH7stmzMfrW6795bOlSjtJ9yAd7C/HbDw5i3qhY/P728fDz4/JEM2Oh+7L0dMfSxbaWLuWcug/YeKQUj7+7D1NS+uLluyYjkA+qMD3+H/R1zu12L/HVVyx1C9uVcxY/XpmJkfHheO2+VIQE+nf9k8jwWOjUealz+sVyMnPPYvHynRjQJxQr7p+C8BDunmgVLHRySEu7dOULwOkXi8nMPYvvv74TcREheOfBaegXFqw7ErkRC50uSk9vv9Q5/WIJmblncd/yXY4yf2ga4vg8UMthodM3dXShtGX6hVMwptRS5jHhwSxzC+OuO3SptDRHsbfdEmDp0m++h0xh68kzeHBFBmKd0ywsc+viCJ3a19H0C8C16iby2aFiLH5jFwb0CcXqh6ahfyTL3MpY6NSxjqZfAF4sNYG1mQX4yardGB0fgXd/NJ0jcx/AQqfOdbSkEeDFUgN7/etT+NV7+zB9cD+s+uFURPUK0h2JvIBz6NS11tMrrefRgYul7twbhvRqblb43adH8PdN2bhxXH/86Y6JCA7gTUO+goVO3dNS6u1dLGWpG0Jtow2/WLMX6w+V4PvTB2LJzWPgz71ZfAqnXOjydLVWnRdLtSiprMcdf9+Ozw6XYMnNo/E/t4xlmfsgjtDp8qWnO8q7vZF6yzEWu9ccLqrEAyscj4179d5UzBsdpzsSaeLSCF1ErheRoyJyQkSe6vpnkGVwWaMhvL+nELe+sgVKAe/+aDrL3Mf1uNBFxB/AXwDcAGA0gDtFZLS7gpEJdFXqXAHjMY22Ziz54CB+vmYvxg+IwoePzOQzQMmlEfoUACeUUtlKqUYAqwHc4p5YZBqdrVXnvLpHnK6ow6Jl27BiWy5+eNUgrHpwKmK5xpzg2hz6AAD5rV4XAJjqWhwypZbCbrukEeC8upttyCrBk2v3o67JjpfvmoSbxifojkQG4soIvb1L6OqSN4k8JCIZIpJRVlbmwunI0DrafrcFp2BcUtdox3+9fwAPrMhATHgwPnxkJsucLuFKoRcASGr1OhFAUds3KaWWKaVSlVKpMTExLpyODC89HVCq42LnFEyPHCiowIKXNmPl9jw8NGswPnhkJobGhuuORQbkypTLLgDDRGQQgEIAiwDc5ZZUZG7p6YB0sAa6ZQqmZekjy71D9U12/DX9JP668QSiw4Lx9g+nYsbQaN2xyMB6XOhKKZuIPAJgPQB/AMuVUofclozMreVCaXvz6sA359YBFnsb27PL8fS/DiC7rAbfnpiAtIVjuB8LdcmlG4uUUh8D+NhNWchKWhd0e9sFtGhb+D5e7OdrG/Hsx0ewJiMfSX1D8Y8fTMGs4ZyqpO4RpS65jukxqampKiMjw2vnIwNJS+t4tN7a7Nk+uSdMo60ZK7fn4s9fHkdVvQ0PXj0Yj80dhtAgbqxFgIhkKqVSu3ofb/0n7+joKUhttTzqzkfm2JVSWH+oGM99cgQ55bW4amg0frNgFEbFR+iORibEETp5V2dr1tsze7Yli10phS0nyvHihmPYlXMOw2LD8PSCUZgzPAbS0QVl8lndHaGz0EmPOXMcH5dT7IDpp2OUUtiQVYqXNp7AvvzziIsIxqNzh+GO1CQE+HPzU2ofp1zI2FoXc3dKvWWqZs4cICcHWLzYVKP2+iY7/r2vCK9/fQpHiquQ1DcUz3xnHG67YgAfQEFuwxE66Xe50zAtBg4EUlIcBZ+T495MbnKyrBqrtudhbWY+KuttGBobhofnDMHCCQkckVO3cYRO5tHdJY5t5eY6PgBHsQPA+fNAVJTWgi+trMcnB4vx0f7T2JlzFoH+gvlj+uOeaQMxdVBfzpGTx7DQyThair27K2Jaayl2AKiocJQ6ANTXOz7bbEBAwMXXbqQbHavmAAAFQElEQVSUwsmyGmw+XoZPDhZjV85ZKAUMjwvDE/NH4HupSYgJD3b7eYnaYqGT8bhS7C0qKi49Zrd3vCUB4NiHphvszQqnzlRjT955bD1Zjq0nz6CksgGAo8QfmzsMC8bFY1gc91sh72Khk3G1LXagZ+XeA/ZmhbKqBhRV1KG4oh5F5+twsqwah4sqcbSkCvVNzQCAfr2DMH1IP8wYEo0ZQ/ohJbq3V/IRtYeFTsbXeo69ZZVL6ykWN5n+7AbUN9lR39SMepv9kgF7VK9AjI6PwN1TB2J0fATGDojEsNgw+PFhzGQQLHQyl5aReloa8OabF1e5uKHgtz09D4BjU/8/fXYUseHBSIgKQf+IUCREhSAyNJAXNMnQuGyRrKH1Kpf25s+7w4t/FoguR3eXLXIhLFlDy1r08+cdW/dGRjo+goMdH/68eYesj4VO1pOW5ij28+cdyxTr6x3LFjt7RB6RBXAOnXyHyfeBIeoKR+hERBbBQicisggWOhGRRbDQiYgsgoVORGQRXr2xSETKAPT0lr5oAGfcGMfTzJSXWT3HTHnNlBUwV15Xsw5USsV09SavFrorRCSjO3dKGYWZ8jKr55gpr5myAubK662snHIhIrIIFjoRkUWYqdCX6Q5wmcyUl1k9x0x5zZQVMFder2Q1zRw6ERF1zkwjdCIi6oSpCl1Evisih0SkWUQMeXVbRK4XkaMickJEntKdpzMislxESkXkoO4sXRGRJBHZKCJZzt8Dj+nO1BkRCRGRnSKyz5l3qe5MXRERfxHZIyL/0Z2lKyKSIyIHRGSviBj6IQsiEiUia0XkiPP373RPnctUhQ7gIIBbAWzSHaQ9IuIP4C8AbgAwGsCdIjJab6pOvQnget0huskG4HGl1CgA0wD81ODf2wYA1yqlJgCYCOB6EZmmOVNXHgOQpTvEZbhGKTXRBEsXXwTwqVJqJIAJ8OD32FSFrpTKUkod1Z2jE1MAnFBKZSulGgGsBnCL5kwdUkptAnBWd47uUEqdVkrtdn5dBccfigF6U3VMOVQ7XwY6Pwx7wUpEEgEsAPCa7ixWIiIRAGYBeB0AlFKNSqnznjqfqQrdBAYAyG/1ugAGLh2zEpEUAJMA7NCbpHPOKYy9AEoBfK6UMnLeFwA8CaBZd5BuUgA+E5FMEXlId5hODAZQBuAN53TWayLS21MnM1yhi8gXInKwnQ/DjnRbae8JwoYdlZmRiIQB+CeAnyulKnXn6YxSyq6UmgggEcAUERmrO1N7ROQmAKVKqUzdWS7DTKXUZDimN38qIrN0B+pAAIDJAF5RSk0CUAPAY9fWDPfEIqXUPN0ZXFAAIKnV60QARZqyWI6IBMJR5quUUut05+kupdR5EUmH43qFES9AzwSwUERuBBACIEJEViql7tGcq0NKqSLn51IR+Rcc051GvLZWAKCg1b/O1sKDhW64EbrJ7QIwTEQGiUgQgEUAPtScyRJEROCYh8xSSv1Rd56uiEiMiEQ5vw4FMA/AEb2p2qeU+rVSKlEplQLH79kvjVzmItJbRMJbvgbwLRjzL0oopYoB5IvICOehuQAOe+p8pip0EfmOiBQAmA7gIxFZrztTa0opG4BHAKyH46Ldu0qpQ3pTdUxE3gGwDcAIESkQkQd0Z+rETAD3ArjWuVRtr3NEaVTxADaKyH44/qL/XCll+OWAJhEH4GsR2QdgJ4CPlFKfas7UmZ8BWOX8vTARwDOeOhHvFCUisghTjdCJiKhjLHQiIotgoRMRWQQLnYjIIljoREQWwUInIrIIFjoRkUWw0ImILOL/A0z7Wm7n1XE0AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "theta_history = []\n",
    "gradient_descent(0.0, 0.01)\n",
    "plot_theta_history()\n",
    "print(len(theta_history))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3682\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl4VdW9PvD3m4EkkAnIQEISEghTGAIYmUUQLA6odWjFqWKttHWotl69tndQ7+3V/no7ONYWFeQKCkqtUidUJAFRhoRBhgQIITMZCGQgIcM5Z/3+SGIpJifDGdbe+7yf58lDcrLJfoX4ZrH22muLUgpERGR+froDEBGRe7DQiYgsgoVORGQRLHQiIotgoRMRWQQLnYjIIljoREQWwUInIrIIFjoRkUUEePNkUVFRKjk52ZunJCIyvZycnFNKqeiejvNqoScnJyM7O9ubpyQiMj0RKerNcZxyISKyCBY6EZFFsNCJiCyChU5EZBEsdCIii2ChExFZBAudiMgiTFHoH3x9Emt39moZJhGRoZTVnsNvPspDVUOzx89likL/8MBJ/G7TEbTY7LqjEBH1yfrdJfjL1uNotTk8fi5TFPrNFyfiTFMbNh2q1B2FiKjX7A6Ft7NLcMnoaCQMHujx85mi0OemRiFhcAjW7y7WHYWIqNe2Hq3Gybpm3HJxolfOZ4pC9/MT3JyRiO35NSiqadQdh4ioV97cVYyo0AFYOD7WK+czRaEDwE0ZCfCT9vkoIiKjq6pvxua8Ktw4LQEDArxTtaYp9LiIECwYG4O3c0phs3v+4gIRkSs27CmF3aFws5emWwATFToALJ2ehOqGFnyeV6U7ChFRtxwOhfW7SzAjZQhGRod67bymKvQFY6MRExaEdZx2ISID21FQg6KaJiyd7r3ROWCyQg/w98P3MhKQeaQKJ+vO6Y5DRNSlN3eXIDw4AFdOjPPqeU1V6ABwc0YSHAp4O7tUdxQiom8509iKTQcrcMO0BAQH+nv13KYr9KShAzE3NQrrd5fA4VC64xAR/ZN39pah1e7w6sXQTqYrdABYOj0RZbXnsC3/lO4oRETfUEph3a5ipCdGYnxcuNfPb8pCvzwtFoMHBmLdLt45SkTGsaf4DI5VnfXanaEXMmWhBwX448ZpCfj0cCVOnW3RHYeICACwblcJBg7wx5L0eC3nN2WhA+3TLjaHwl9zeHGUiPRraG7D+1+fxLXp8QgNCtCSwbSFnhoThowRg7F+dwmU4sVRItLrvX3lONdmx9LpSdoymLbQgfY7RwtONWJHwWndUYjIhymlsG53McYNC0N6QoS2HD0WuoisFJEqETl43mtDRORTETnW8etgz8bs2tWT4hAeHMCnGRGRVvtL63CwrB63TE+CiGjL0ZsR+msArrjgtccAbFZKjQawueNjrwsZ4I+bLkrEpkMVqG7gxVEi0mPNjiIMHOCPG6YN15qjx0JXSm0FcOGcxnUAVne8vxrAd92cq9dum5mENrvCW9nc34WIvK+2qRV/31+O66YMR1hwoNYs/Z1Dj1VKnQSAjl9jujtQRJaLSLaIZFdXV/fzdN0bFR2K2aOG4o2dxbDzzlEi8rINOaVosTlw+0x9F0M7efyiqFJqhVIqQymVER0d7ZFz3DFzBMpqz2ELt9UlIi9yOBTW7izGtKRITIjXdzG0U38LvVJE4gCg41etTbooLRYxYUFYw4ujRORFXx6vwYlTjbh95gjdUQD0v9A3Ariz4/07Abznnjj9E+jvh6XTk5B1tBrFNU06oxCRD1mzowiDBwbiqkne3Sa3O71ZtvgmgK8AjBWRUhG5G8BvAFwuIscAXN7xsVa3TE+EnwjW7uIonYg8r6KuGZ/mVuL7GYle3ya3Oz3en6qUuqWbTy10cxaXxEWEYNH4GLydXYpfXD4GQQHG+AMmImt6c1f7QoxbZ+i/GNrJ1HeKXuj2mSNwurEVHx2o0B2FiCysze7Aut3FuHRMNEYMHaQ7zjcsVehzRkUheehArNnBaRci8pzNuZWorG8xzMXQTpYqdD8/we0zRyC76AxyT9brjkNEFvX6jiLERwTjsnHd3oKjhaUKHQBuuigBQQF+HKUTkUcUVJ/F9vwa3DojCf5++vZt6YrlCj1y4AAsmRyPd/eWoaG5TXccIrKYtTuLEeAn+L6mpxI5Y7lCB4A7Zo1AY6sd7+4t0x2FiCzkXKsdb2eXYPHEYYgJC9Yd51ssWejpCRGYODwcr+8o4sMviMht/r6/HPXNNtw+w1gXQztZstBFBD+YlYyjlWfx1fEa3XGIyAKUUlj1ZSHGxoZh5sghuuN0yZKFDgDXpsdjyKABWPVloe4oRGQBu06cRu7Jeiybk6z1IRbOWLbQgwP9cev0JHyWW4mS09zfhYhc89qXhYgcGIjvTtH7EAtnLFvoQPudo34iWM1ROhG5oPRMEzYdqsDSi5MQMsC424pYutCHRQTjyonDsD67BI0tNt1xiMikXt9RBBHBHbOMeTG0k6ULHQDumpOChmYb3tlTqjsKEZnQuVY71u0qweIJsRgeGaI7jlOWL/RpSZGYnBCB174shIOPqCOiPvrb3jLUnWvDstkpuqP0yPKFLiK4a04yjlc34ov8U7rjEJGJKKXw2pcnMCE+HBcnD9Ydp0eWL3QAuGpSHKJCg7Bq+wndUYjIRL46XoOjlWexbLZxlyqezycKPSjAH7fNSMKWI9U4capRdxwiMomV2wsxdNAAXJMerztKr/hEoQPAbTOTEOjPJYxE1DvFNU3YnFeJW2ckGeYRcz3xmUKPCQvGksnxeDu7hLswElGP/u+rQviL4DaD7tvSFZ8pdABYNjsZja12bMjhEkYi6l5jiw3rs0tw5aQ4DIsw3q6K3fGpQk9PjMS0pEis5hJGInLinT2laGi24a45ybqj9IlPFToALJuTgsKaJnyeV6U7ChEZkMPRvqtiekIEpiZG6o7TJz5X6FdOHIa4iGC88kWB7ihEZEBbjlShoLoRP5ybYoqliudzqdBF5OcickhEDorImyJi+MmmQH8//HBOCnYUnMbXpbW64xCRwazYWoDhkSG4alKc7ih91u9CF5HhAH4GIEMpNRGAP4Cl7grmSUunJyIsKAAvb+ONRkT0D/tLarHzxGncNScZgf7mm8BwNXEAgBARCQAwEEC565E8Lyw4ELfMSMKHB05yr3Qi+sbL2woQFhyApdOTdEfpl34XulKqDMDvABQDOAmgTin1ibuCedqy2ckQAKu2F+qOQkQGUHK6CR8eOIlbpychNChAd5x+cWXKZTCA6wCkAIgHMEhEbu/iuOUiki0i2dXV1f1P6mbxkSG4Jj0e63YXo66JNxoR+bqV20/ATwTLTLZU8XyuTLksAnBCKVWtlGoD8A6A2RcepJRaoZTKUEplREdHu3A69/vRJSloarXjjV3FuqMQkUZ1TW1Yv7sE16bHIy7C2HueO+NKoRcDmCkiA6V9bc9CALnuieUdE+IjMDc1Cqu2n0CrzaE7DhFpsnZXEZpa7fjRJSN1R3GJK3PoOwFsALAHwIGOr7XCTbm85p55I1HV0IKN+01xPZeI3KzFZsdr2wtxyegopMWH647jEpdWuSilHldKjVNKTVRK3aGUanFXMG+ZNzoKY2PD8Mq2AijF7QCIfM3GfeWoamjBPSYfnQM+eKfohUQE98wbibyKBmw7xicaEfkSpRRe3laAccPCcMnoKN1xXObzhQ4A16bHIzY8CC9v43YARL4k62g1jlaexT2XjDTdbf5dYaEDGBDgh2WzU7Dt2CkcLq/XHYeIvOTlbQWIDQ8yzROJesJC73DrjCQMGuDPUTqRjzhYVoft+TW4a04KBgRYowqt8V/hBhEhgbj54iRs3F/O7QCIfMCfs44jNCgAt5j0Nv+usNDPc8+8FPgJOEonsriC6rP44MBJ3D5zBCJCAnXHcRsW+nniIkJw47QErNtdgqqGZt1xiMhD/pJVgAH+frh7boruKG7FQr/Ajy8dBZvdgZVfFOqOQkQeUF57Du/sLcXNFyciOixIdxy3YqFfICVqEK6eHI81O4q4aReRBb28rQBKAcvnmf9Gogux0Ltw7/xRONtiw/99Vag7ChG5Uc3ZFry5qxjXTRmOhMEDdcdxOxZ6F8bHhWPhuBis3H4CTa023XGIyE1WbS9Ei82Bn8633ugcYKF3694FqTjT1IY3d5XojkJEblDf3IbVXxXiignDkBoTpjuOR7DQu3HRiMGYOXIIVmw9jhabXXccInLRmh1FaGi24b4FqbqjeAwL3Yn7FqSisr4F7+wp0x2FiFxwrtWOV7edwKVjojFxeITuOB7DQndibmoUJidE4M9Zx2Gz8wEYRGa1fncxahpbLT06B1joTokI7p2fiqKaJnxw4KTuOETUD602B1ZsLcDFyYMxPWWI7jgexULvwXfSYpEaE4o/bTkOh4MPwCAym3f3laG8rhn3Wnx0DrDQe+TnJ7h3/igcqWzA5rwq3XGIqA/sDoU/Zx5HWlw45o8x1kPqPYGF3gvXpMdjxNCBeHbzUT6mjshENu4vQ8GpRjxwWaolHmDRExZ6LwT6++H+Bak4WFaPTw9X6o5DRL1gszvw3OZ8jBsWhsUThumO4xUs9F66fupwJA8diGc+O8ZROpEJvLevHCdONeKhRWPg52f90TnAQu+1AH8/PHDZaBw+WY9NhzhKJzIym92B5z8/hrS4cCyeEKs7jtew0PvguinxSIkahGc+O8oVL0QG9u6+chTWNOGhRaN9Yu68k0uFLiKRIrJBRPJEJFdEZrkrmBG1j9JTkVfRgE2HKnTHIaIudI7OJ8SH4/I03xmdA66P0J8F8LFSahyAdAC5rkcytmvT4zEyahCe+ewYR+lEBvTO3jIU1TThoUVjfGp0DrhQ6CISDmAegFcBQCnVqpSqdVcwowrw98PPFo7GkcoGfHSQo3QiI2nrGJ1PGh6BReNjdMfxOldG6CMBVANYJSJ7ReQVERnkplyGdk16PEZFD8KzmzmXTmQk7+wpRcnpcz43d97JlUIPADANwEtKqakAGgE8duFBIrJcRLJFJLu6utqF0xmHv5/gZwtH42jlWe7xQmQQrTYHnv88H+kJEbhsnO+NzgHXCr0UQKlSamfHxxvQXvD/RCm1QimVoZTKiI62zq23SybHIzUmFM9uPgY7R+lE2v11TylKz5zzybnzTv0udKVUBYASERnb8dJCAIfdksoE/P0EDy4cjfyqs3j/63LdcYh8WqvNgRc+z8eUxEjMH2udgWNfubrK5QEAa0XkawBTADzleiTzuHpSHMbEto/SuV86kT5v55SgrNZ35847uVToSql9HdMpk5VS31VKnXFXMDPw8xP8fNEYFFQ34p29fKoRkQ7NbXY8t/kYLhoxGJf6wI6KzvBOURddMXEYJidE4NnPjvHZo0QarP6yEJX1LXh08VifHp0DLHSXiQgeXTwOZbXnsHZHse44RD6l7lwb/pR5HPPHRmPGyKG642jHQneDuaOjMHvUULy4JR9nW2y64xD5jJe3FqDuXBv+5Ttjez7YB7DQ3eSRxWNR09iKlV+c0B2FyCdUN7Rg5fYTWDI5DhOHR+iOYwgsdDeZmjQY30mLxYqtBTjd2ArMn687EpGlvbglHy02Bx7m6PwbLHQ3+pfFY9HUasNLmflAVpbuOESWVXK6CWt3FuH7GYlIifKJHUd6hYXuRmNiw/DToc1YeN8t7S9wlE7kEX/87Cj8pP3mPvqHAN0BLGX+fDxy/sg8KwsQAS69FMjM1BaLyEoOl9fjb3vLsPySkRgWEaw7jqFwhE5EpvL0R7kIDw7EvfNTdUcxHBa6O3U3xZKVxekXIjfYerQa246dwgOXpSJiYKDuOIbDQnenJ55on14hIrezOxSe+jAXiUNCcMesEbrjGBIL3d04SifyiL/tLUNeRQMeXTwOQQH+uuMYEgvd3ThKJ3K75jY7fv/JEaQnRmLJ5DjdcQyLhe4JmZnAiC7+SZiVBSQnezsNkem9+sUJnKxrxq+uHOfzG3A5w0L3lO6Ku6iofRRPRL1Sc7YFf848jkXjY7kBVw9Y6J6Smdn91AvXpBP12u8/PYpzbXY8duU43VEMj4WuQ1YWR+lEvZB7sh7rdhXjjlkjkBoTqjuO4bHQPam7uXQAeO01byYhMh2lFP7r74cRHhLIW/x7iYXuacuWdf16URGXMRI58cnhSnxVUINfXD4GkQMH6I5jCix0T+MyRqI+a7HZ8dSHuRgdE4pbpyfpjmMaLHRv4M1GRH2yanshimqa8B9L0hDgz5rqLf5JecMTT3Q/l15Y6M0kRIZX3dCCFz7Px8JxMZg3Jlp3HFNhoXtLYWHXpV5UxJuNiM7z24/z0Nxmx79dPV53FNNhoXsTbzYiciqn6AzezinF3XNTMDKayxT7yuVCFxF/EdkrIu+7I5ClcRkjUbfsDoX/fO8gYsOD8ACXKfaLO0boDwLIdcPX8Q1cxkjUpTd2FuFQeT3+/eo0hAbxYWr94VKhi0gCgKsBvOKeOD6AF0iJvqXmbAv+d9MRzB41lLspusDVEfozAB4F4OjuABFZLiLZIpJdXV3t4uksghdIif7Jbz8+gqZWO568dgJ3U3RBvwtdRJYAqFJK5Tg7Tim1QimVoZTKiI7mEqRv8AIpEQBgT/EZrM8uwV1zkjE6Nkx3HFNzZYQ+B8C1IlIIYB2Ay0RkjVtS+QJeICWCze745kLog4vG6I5jev0udKXUL5VSCUqpZABLAXyulLrdbcl8gbMLpJx6IR+w+qsiHCzjhVB34Tp0nZzt88KpF7K48tpz+P0nRzB/bDQvhLqJWwpdKZWplFrijq/lczj1Qj7q8Y2H4FAK/33dRF4IdROO0I2AF0jJx3x8sAKfHq7EzxeNQeKQgbrjWAYL3QgyM4GIiK4/98wzXo1C5GkNzW14YuMhjBsWhh/OTdEdx1JY6EZRW9v11EtdHS+QkqX8/pOjqGxoxtM3TEIgt8Z1K/5pGomzqRduC0AWsLf4DFZ/VYg7Zo7A1KTBuuNYDgvdSJxdIOW2AGRyLTY7Ht3wNYaFB+ORxWN1x7EkFrrRcG06WdTzm/NxrOosnrphEsKCA3XHsSQWutFwbTpZ0MGyOryUdRw3TkvAgrExuuNYFgvdiLjqhSykze7AIxu+xpBBA/AfS/gUIk9ioRvVQw91/TpXvZDJvJR5HLkn6/Hr705E5MABuuNYGgvdqJztm86pFzKJIxUNeP7zY7gmPR6LJwzTHcfyWOhG1t2+6QC3BSDDa7U58PDb+xAWHIgnrknTHccnsNCNjqteyKSe//wYDpbV46nrJ2FoaJDuOD6BhW50nHohE9pTfAYvbsnHjdMScMVETrV4CwvdDAoLueqFTKOp1YaH39qPuIgQPH4tp1q8iYVuFlz1Qibx9Id5OHGqEb/7XjrCeQORV7HQzaKnqRfu9UIGkHW0Gq/vKMLdc1Mwa9RQ3XF8DgvdTJxNvezY4dUoRBc63diKR97ej9ExodyrRRMWutnU1gJBXawYaGkBIiO9n4cIgFIKj7y9H7VNbXhm6RQEB/rrjuSTWOhmNHNm169zPp00ee3LQmzOq8KvrhqHCfHd/CuSPI6FbkbOttnlUkbyskPldXj6wzwsGh+DO2cn647j01joZlVY2PXUCwD8+tdejUK+q6nVhgfe3IvBgwLx25vS+bBnzVjoZtbcDPh3MVdpt3M+nbziiY2HcOJUI/548xQMGcSNt3RjoZtdQkLXr9fVcSkjedS7e8vwVnYp7l+QitmjonTHIbhQ6CKSKCJbRCRXRA6JyIPuDEa95GwpY1YW59PJI45WNuCX7xzA9JQheHDhaN1xqIMrI3QbgIeVUuMBzARwn4jwPl8dulvKCHA+ndzubIsNP1mTg0FBAXjhlqkI8Oc/9I2i338TSqmTSqk9He83AMgFMNxdwaiPHnus69c5n05upJTCv274GkU1TXjh1qmICQ/WHYnO45YfrSKSDGAqgJ3u+HrUD86eRcr16eQmK7cX4oMDJ/HI4rGYOZK39huNy4UuIqEA/grgIaVUfRefXy4i2SKSXV1d7erpyBlnzyLl+nRyUU7RaTz9YS4uT4vFj+eN1B2HuiBKqf7/ZpFAAO8D2KSU+kNPx2dkZKjs7Ox+n496KSCgfaqlKy78fZPvqqhrxjUvfIGBA/yx8f65iAjhLoreJCI5SqmMno5zZZWLAHgVQG5vypy8yGbr/nO88YP6qLnNjuWvZ6OpxYaXf5DBMjcwV6Zc5gC4A8BlIrKv4+0qN+UiVz3+ePefC+aFLOodpRQe++vXOFBWh2eWTsWY2DDdkcgJV1a5fKGUEqXUZKXUlI63D90ZjlzgbP/0lhZeJKVeWbG1AO/uK8fDl4/B5WmxuuNQD7iA1Mqc3XTEh2JQD7YcqcJvPs7D1ZPjcN+CVN1xqBdY6FZXW9v1fi8A7ySlbh2paMDP3tiL8cPC8b83TeamWybBQvcFzi6SPvmk93KQKVTWN+OuVbsQMsAfr9yZgYEDAnRHol5iofsKZxdJOfqiDo0tNvzwtd2oO9eGlcsuRnxkiO5I1AcsdF/h7E5SgKVOsNkduP+NPciraMALt03DxOF88pDZsNB9SWam81IP4D+tfZVSCo9vPIQtR6rxX9dNwIKxMbojUT+w0H1NZmb3OzNyIy+f9eKWfKzdWYyfXDoKt83oZrkrGR4L3Rd196QjoH0jL5a6T3l9RxF+98lRXD91OB5dPFZ3HHIBC91X2Wzdz5tzd0af8d6+MvznewexaHwMfnvTZPj58VqKmbHQfZnD0f3nuDuj5W3Jq8LDb+3H9OQheOHWaQjkgypMj3+Dvs7Z7otPPslSt6jdhafxkzU5GBcXhlfuzEBwYDdTcGQqLHRiqfuYnKLTWLZyF4YPDsHqu6YjLJi7J1oFC53asdR9Qk7Rafzg1V2IDQ/Gm/fMxNDQblY8kSmx0OkfWOqWllN0Gneu3N1e5stnIpbPA7UcFjr9s55KnTs0mlJnmUeHBbHMLYyFTt/mrNSzsrik0WS+PH4KP3h1V3uZ38MytzIWOnXNWakXFfHmI5P45FAFlq3ajeGDQ7Bu+UwMi2CZWxkLnbrnrNTr6gA/fvsY2YacUvx07R6kxYXjrR/P4sjcB3A3JnJOqe7vKO38nLPiJy1e/eIE/vv9w5ibGoW/3HERBgXxf3VfwL9l6pmzUgdY6gbicCj8v4/z8JetBbhq0jD88eYpCArgTUO+goVOvcNSN7ymVht+vn4fNh2qxA9mjcDj10yAP/dm8SksdOq93pT6iBHtD6cmr6qsb8aPVmfjYHkdHr8mDXfNSdEdiTRgoVPf9FTqRUXtD8pw9hxTcqvD5fW4e3X7Y+NeviMDi9JidUciTVxapiAiV4jIERHJF5HH3BWKDK6nqRW7nY+085J395bhhpe2QyngrR/PYpn7uH4Xuoj4A3gRwJUA0gDcIiJp7gpGBqdU908+6iTCm5A8pNXmwOPvHcRD6/dh8vBIbLx/Dp8BSi5NuUwHkK+UKgAAEVkH4DoAh90RjEygubn9BqO6uu6PKSpqX6/ubO916pOTdedw39o92FNcix/NTcG/XjmOe5kTANemXIYDKDnv49KO18iX1NY6f/A08I95d27u5bLNuZVY8twXyKtowAu3TsW/L0ljmdM3XPlO6GqS9FuTqyKyXESyRSS7urrahdORYWVm9m7J4pNPcm69n8612vHv7x7A3auzER0WhI33z8GSyfG6Y5HBuFLopQASz/s4AUD5hQcppVYopTKUUhnR0dEunI4Mr7fr0EW4a2MfHCitw9XPb8OaHcVYPm8k3rt/DlJjwnTHIgNyZQ59N4DRIpICoAzAUgC3uiUVmZdSQHAw0NLi/LisLN6M1IPmNjv+lHkcf9qSj6jQILzxoxmYnRqlOxYZWL9H6EopG4D7AWwCkAvgLaXUIXcFIxNrbgYef7x3x4pwGqYLOwpqcNVz2/Dc5mNYMjkOHz90CcuceiTKiyOkjIwMlZ2d7bXzkQH0payDgtp/GPiw2qZWPP1hHtZnlyBxSAj+57uTMG8Mpyp9nYjkKKUyejqOd4qSZ/V2CgZoP0YEiIhoXz3jQ1ptDqzZUYTnPj+GhmYbfnLpKDy4cDRCBnBjLeo9Fjp5Xueou7ej9bq69mP9/S2/hYBSCpsOVeA3H+WhsKYJc1Oj8G9Xj8f4uHDd0ciEWOjkPX0ZrQP/vIWAxS6eKqWwPb8Gz24+it2FZzA6JhSr7roY88dEQ3hNgfqJhU7e1dfReieLFLtSCptzq/D8lnzsL6lFbHgQ/uf6ibg5IxEBvEGIXMRCJz2Uat/npaiob7/v/B8EJir35jY7/r6/HK9+cQJ5FQ1IHBKCp66fhBsvGs4HUJDbsNBJn8590/38+lfOJij349VnsXZHMTbklKC+2YbUmFD84fvpuDY9niNycjsWOunXuXGXK3PHF/5ejQVfVd+Mjw5W4IOvT2JX4WkE+gsWTxiG22eOwIyUIZwjJ49hoZNxdJawOwrP2YOt3UwphePVjdh2rBofHazA7sLTUAoYExuKRxaPxfczEhEd1sNWw0RuwEIn43FnsV/I2dfsZdnbHQonTp3F3uJafHm8Bl8eP4XK+vaVO2NiQ/HgwtG4elIcRsdyvxXyLhY6GZcni70HdodCdUMLyuvOoaKuGeW153C8+iwOl9fjSGUDmtvap4mGDhqAWaOGYvaoKMweNRTJUYO8npWoEwudjO/8kbMHy33W05vR3GZHc5sDzTb7twbskQMDkRYXjttmjEBaXDgmDo/A6JhQ+PlxTpyMgYVO5tLZsv1dGePEV79a1H4KAH/85AhiwoIQHxmMYeEhiI8MRkRIIC9okqGx0Mmczn+kXX/Ws3el4weEAPiF61+NyOu4EJbMr7CwvYw73/x5ow75Jo7QyXq629CL0yVkcSx08h0GvZuUyF045UJEZBEsdCIii2ChExFZBAudiMgiWOhERBYhyotX/kWkGkB/7wCJAnDKjXE8zUx5mdVzzJTXTFkBc+V1NesIpVR0Twd5tdBdISLZSqkM3Tl6y0x5mdVzzJTXTFkBc+X1VlZOuRARWQQLnYjIIsxU6Ct0B+gjM+VlVs8xU14zZQXMldeKPdPQAAADWklEQVQrWU0zh05ERM6ZaYROREROmKrQReR7InJIRBwiYsir2yJyhYgcEZF8EXlMdx5nRGSliFSJyEHdWXoiIokiskVEcju+Bx7UnckZEQkWkV0isr8j75O6M/VERPxFZK+IvK87S09EpFBEDojIPhHJ1p3HGRGJFJENIpLX8f07y1PnMlWhAzgI4AYAW3UH6YqI+AN4EcCVANIA3CIiaXpTOfUagCt0h+glG4CHlVLjAcwEcJ/B/2xbAFymlEoHMAXAFSIyU3OmnjwIIFd3iD5YoJSaYoKli88C+FgpNQ5AOjz4Z2yqQldK5SqljujO4cR0APlKqQKlVCuAdQCu05ypW0qprQBO687RG0qpk0qpPR3vN6D9f4rhelN1T7U72/FhYMebYS9YiUgCgKsBvKI7i5WISDiAeQBeBQClVKtSqtZT5zNVoZvAcAAl531cCgOXjlmJSDKAqQB26k3iXMcUxj4AVQA+VUoZOe8zAB4F4OjpQINQAD4RkRwRWa47jBMjAVQDWNUxnfWKiAzy1MkMV+gi8pmIHOzizbAj3fN09Ugcw47KzEhEQgH8FcBDSql63XmcUUrZlVJTACQAmC4iE3Vn6oqILAFQpZTK0Z2lD+YopaahfXrzPhGZpztQNwIATAPwklJqKoBGAB67tma4JxYppRbpzuCCUgCJ532cAKBcUxbLEZFAtJf5WqXUO7rz9JZSqlZEMtF+vcKIF6DnALhWRK4CEAwgXETWKKVu15yrW0qp8o5fq0Tkb2if7jTitbVSAKXn/etsAzxY6IYboZvcbgCjRSRFRAYAWApgo+ZMliAigvZ5yFyl1B905+mJiESLSGTH+yEAFgHI05uqa0qpXyqlEpRSyWj/nv3cyGUuIoNEJKzzfQDfgTF/UEIpVQGgRETGdry0EMBhT53PVIUuIteLSCmAWQA+EJFNujOdTyllA3A/gE1ov2j3llLqkN5U3RORNwF8BWCsiJSKyN26MzkxB8AdAC7rWKq2r2NEaVRxALaIyNdo/0H/qVLK8MsBTSIWwBcish/ALgAfKKU+1pzJmQcArO34XpgC4ClPnYh3ihIRWYSpRuhERNQ9FjoRkUWw0ImILIKFTkRkESx0IiKLYKETEVkEC52IyCJY6EREFvH/AbCJalLcZb7dAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "theta_history = []\n",
    "gradient_descent(0.0, 0.001)\n",
    "plot_theta_history()\n",
    "print(len(theta_history))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xdc1lX/x/HXhw3KUAEVEEFQFLeRO3eZo3k3LLO0YdP2Xlrdjbu77rLblpVlZWba3sOs1HLg3lsRXLgAQdbF+f0B9vMuB8IF5xqf5+PhQ8BLvm8V3x7P93zPEWMMSiml3J+P7QBKKaWcQwtdKaU8hBa6Ukp5CC10pZTyEFroSinlIbTQlVLKQ2ihK6WUh9BCV0opD6GFrpRSHsKvNi8WGRlpEhISavOSSinl9hYtWrTXGBN1stfVaqEnJCSQnp5em5dUSim3JyLbKvM6nXJRSikPoYWulFIeQgtdKaU8hBa6Ukp5CC10pZTyEFroSinlIbTQlVLKQ7hFoX+9fCdT5ldqGaZSSrmUrIOHeebbtezJK6zxa7lFoX+zYifPfb+OolKH7ShKKXVKpi3czuu/baK4tKzGr+UWhX7p6U04UFDC96t2246ilFKV5igzTE/fzhnNo4irF1Lj13OLQu+ZHElcvWCmLcywHUUppSrtt/XZ7Mwp5LLTm9TK9dyi0H18hEvTmjB34z627cu3HUcppSpl6oIMIusG0L9Vw1q5nlsUOsBFaXH4SPl8lFJKubo9uYXMXLuHf3SKI8CvdqrWbQq9cXgwfVOimb4ok1JHzd9cUEqp6pixOBNHmeHSWppuATcqdIBhnePJzivi57V7bEdRSqnjKiszTFu4nS6J9WkWVbfWrutWhd43JYro0EA+1GkXpZQLm7d5H9v2FTCsc+2NzsHNCt3P14eL0+L4Zd0eduYcth1HKaWOaerC7YQF+TGoTeNava5bFTrApWnxlBmYnp5pO4pSSv3Ngfxivl+5iws7xRHk71ur13a7Qo9vEELP5EimLdxOWZmxHUcppf7HJ0uyKHaU1erN0CPcrtABhnVuQtbBw8zeuNd2FKWU+pMxhg8XZNC+SQStGofV+vXdstDPTG1IvRB/PlygT44qpVzH4owDbNhzqNaeDP0rtyz0QD9f/tEpjh9X72bvoSLbcZRSCoAPF2wnJMCXoe1jrFzfLQsdyqddSssMHy/Sm6NKKfvyCkv4avlOzm0fQ91APysZ3LbQk6NDSWtaj2kLt2OM3hxVStn1+dIdHC5xMKxzvLUMblvoUP7k6Oa9+czbvN92FKWUFzPG8OHCDFo2CqV9XLi1HCctdBGZJCJ7RGTlUR+rLyI/isiGiu/r1WzMYxvStjFhQX56mpFSyqplmTmszMrlss7xiIi1HJUZob8DnP2Xj90PzDTGNAdmVrxf64IDfLnotCZ8v2oX2Xl6c1QpZcf787YREuDLhZ1ireY4aaEbY34D/jqncR4wueLtycD5Ts5VacO7xlPiMHyUrvu7KKVq38GCYr5ctoPzOsQSGuRvNUtV59AbGmN2AlR8H328F4rIaBFJF5H07OzsKl7u+JKi6tI9qQEfzM/AoU+OKqVq2YxFmRSVlnFFV3s3Q4+o8ZuixpiJxpg0Y0xaVFRUjVxjRNemZB08zCzdVlcpVYvKygxT5mfQKT6C1jH2boYeUdVC3y0ijQEqvrfapANSGxIdGsj7enNUKVWLft+0jy1787mia1PbUYCqF/oXwFUVb18FfO6cOFXj7+vDsM7x/Lo+m4x9BTajKKW8yPvztlEvxJ/BbWt3m9zjqcyyxanAH0CKiGSKyDXAM8CZIrIBOLPifasu69wEHxGmLNBRulKq5u3KKeTHNbu5JK1JrW+TezwnfT7VGHPZcX6ov5OzVEvj8GAGtIpmenomd57ZgkA/1/gNVkp5pqkLyhdiXN7F/s3QI9z6SdG/uqJrU/bnF/Ptil22oyilPFiJo4wPF2bQu0UUTRvUsR3nTx5V6D2SIkloEML783TaRSlVc2au2c3u3CKXuRl6hEcVuo+PcEXXpqRvO8Canbm24yilPNR787YREx5Ev5bHfQTHCo8qdICLTosj0M9HR+lKqRqxOfsQczfu4/Iu8fj62Nu35Vg8rtAjQgIY2i6Gz5ZkkVdYYjuOUsrDTJmfgZ+PcImlU4lOxOMKHWBEt6bkFzv4bEmW7ShKKQ9yuNjB9PTtDGzTiOjQINtx/sYjC719XDhtYsN4b942PfxCKeU0Xy7bQW5hKVd0ca2boUd4ZKGLCFd2S2D97kP8sWmf7ThKKQ9gjOHt37eS0jCUrs3q245zTB5Z6ADnto+hfp0A3v59q+0oSikPsGDLftbszGVkjwSrh1iciMcWepC/L5d3juenNbvZvl/3d1FKVc87v28lIsSf8zvYPcTiRDy20KH8yVEfESbrKF0pVQ2ZBwr4ftUuhp0eT3CA624r4tGF3ig8iEFtGjEtfTv5RaW24yil3NR787YhIozo5po3Q4/w6EIHGNUjkbzCUj5ZnGk7ilLKDR0udvDhgu0MbN2Q2Ihg23FOyOMLvVN8BO3iwnnn962U6RF1SqlT9OmSLHIOlzCye6LtKCfl8YUuIozqkcCm7HzmbNxrO45Syo0YY3jn9y20jgnj9IR6tuOclMcXOsDgto2JrBvI23O32I6ilHIjf2zax/rdhxjZ3XWXKh7NKwo90M+X4V3imbUumy17823HUUq5iUlzt9KgTgDntI+xHaVSvKLQAYZ3jcffV5cwKqUqJ2NfATPX7ubyLvEuc8TcyXhNoUeHBjG0XQzT07frLoxKqZN694+t+Iow3EX3bTkWryl0gJHdE8gvdjBjkS5hVEodX35RKdPStzOobWMahbverorH41WF3r5JBJ3iI5isSxiVUifwyeJM8gpLGdUjwXaUU+JVhQ4wskciW/cV8PPaPbajKKVcUFlZ+a6K7ePC6dgkwnacU+J1hT6oTSMahwfx5pzNtqMopVzQrHV72Jydz9U9E91iqeLRqlXoInKHiKwSkZUiMlVEXH6yyd/Xh6t7JDJv836WZx60HUcp5WIm/raZ2IhgBrdtbDvKKatyoYtILHArkGaMaQP4AsOcFawmDevchNBAP96YrQ8aKaX+37LtB5m/ZT+jeiTg7+t+ExjVTewHBIuIHxAC7Kh+pJoXGuTPZV3i+WbFTt0rXSn1pzdmbyY0yI9hneNtR6mSKhe6MSYLeA7IAHYCOcaYH5wVrKaN7J6AAG/P3Wo7ilLKBWzfX8A3K3Zyeed46gb62Y5TJdWZcqkHnAckAjFAHRG54hivGy0i6SKSnp2dXfWkThYTEcw57WP4cGEGOQX6oJFS3m7S3C34iDDSzZYqHq06Uy4DgC3GmGxjTAnwCdD9ry8yxkw0xqQZY9KioqKqcTnnu/aMRAqKHXywIMN2FKWURTkFJUxbuJ1z28fQONy19zw/keoUegbQVURCpHxtT39gjXNi1Y7WMeH0TI7k7blbKC4tsx1HKWXJlAXbKCh2cO0ZzWxHqZbqzKHPB2YAi4EVFZ9ropNy1ZrrejVjT14RXyxzi/u5SiknKyp18M7crZzRPJLUmDDbcaqlWqtcjDFjjTEtjTFtjDEjjDFFzgpWW3o1jySlYShvzt6MMbodgFLe5oulO9iTV8R1bj46By98UvSvRITrejVj7a48Zm/QE42U8ibGGN6YvZmWjUI5o3mk7TjV5vWFDnBu+xgahgXyxmzdDkApb/Lr+mzW7z7EdWc0c7vH/I9FCx0I8PNhZPdEZm/Yy+odubbjKKVqyRuzN9MwLNBtTiQ6GS30Cpd3iadOgK+O0pXyEiuzcpi7cR+jeiQS4OcZVegZvwonCA/259LT4/li2Q7dDkApL/Dar5uoG+jHZW76mP+xaKEf5bpeifgIOkpXysNtzj7E1yt2ckXXpoQH+9uO4zRa6EdpHB7MPzrF8eHC7ezJK7QdRylVQ17/dTMBvj5c0zPRdhSn0kL/i+t7J1HqKGPSnK22oyilasCOg4f5ZEkml57ehKjQQNtxnEoL/S8SI+swpF0M78/bppt2KeWB3pi9GWNgdC/3f5Dor7TQj+GmPkkcKirl3T+22o6ilHKifYeKmLogg/M6xBJXL8R2HKfTQj+GVo3D6N8ymklzt1BQXGo7jlLKSd6eu5Wi0jJu7ON5o3PQQj+um/omc6CghKkLttuOopRygtzCEib/sZWzWzciOTrUdpwaoYV+HKc1rUfXZvWZ+NsmikodtuMoparp/XnbyCss5ea+ybaj1Bgt9BO4uW8yu3OL+GRxlu0oSqlqOFzs4K3ZW+jdIoo2seG249QYLfQT6JkcSbu4cF77dROlDj0AQyl3NW1hBvvyiz16dA5a6CckItzUJ5lt+wr4esVO23GUUlVQXFrGxN82c3pCPTon1rcdp0ZpoZ/EWakNSY6uyyuzNlFWpgdgKOVuPluaxY6cQm7y8NE5aKGflI+PcFOfJNbtzmPm2j224yilToGjzPDaL5tIbRxGnxaudUh9TdBCr4Rz2sfQtEEI42eu12PqlHIjXyzLYvPefMb0S/aIAyxORgu9Evx9fbilbzIrs3L5cfVu23GUUpVQ6ijjpZkbadkolIGtG9mOUyu00Cvpgo6xJDQI4cWfNugoXSk38PnSHWzZm8/tA1rg4+P5o3PQQq80P18fxvRrzuqduXy/SkfpSrmyUkcZ//15A6mNwxjYuqHtOLVGC/0UnNchhsTIOrz403pd8aKUC/ts6Q627ivg9gHNvWLu/IhqFbqIRIjIDBFZKyJrRKSbs4K5ovJRejJrd+Xx/apdtuMopY7hyOi8dUwYZ6Z6z+gcqj9CHw98Z4xpCbQH1lQ/kms7t30MzSLr8OJPG3SUrpQL+mRJFtv2FXD7gBZeNTqHahS6iIQBvYC3AIwxxcaYg84K5qr8fH24tX9z1u3O49uVOkpXypWUVIzO28aGM6BVtO04ta46I/RmQDbwtogsEZE3RaSOk3K5tHPax5AUVYfxM3UuXSlX8sniTLbvP+x1c+dHVKfQ/YBOwKvGmI5APnD/X18kIqNFJF1E0rOzs6txOdfh6yPc2r8563cf0j1elHIRxaVl/PfnjbSPC6dfS+8bnUP1Cj0TyDTGzK94fwblBf8/jDETjTFpxpi0qCjPefR2aLsYkqPrMn7mBhw6SlfKuo8XZ5J54LBXzp0fUeVCN8bsAraLSErFh/oDq52Syg34+gi39W/Oxj2H+Gr5DttxlPJqxaVlTPh5Ix2aRNAnxXMGjqequqtcxgBTRGQ50AF4qvqR3MeQto1p0bB8lK77pStlz/RF28k66L1z50dUq9CNMUsrplPaGWPON8YccFYwd+DjI9wxoAWbs/P5ZImeaqSUDYUlDl6auYHTmtajtxfsqHgi+qRoNZ3dphHt4sIZ/9MGPXtUKQsm/76V3blF3DswxatH56CFXm0iwr0DW5J18DBT5mXYjqOUV8k5XMIrv2yiT0oUXZo1sB3HOi10J+jZPJLuSQ14edZGDhWV2o6jlNd447fN5Bwu4e6zUk7+Yi+ghe4k9wxMYV9+MZPmbLEdRSmvkJ1XxKS5WxjarjFtYsNtx3EJWuhO0jG+HmelNmTib5vZn18M48bZjqSUR3t51kaKSsu4S0fnf9JCd6K7B6ZQUFzKq79shMcesx1HKY+1fX8BU+Zv45K0JiRGesWOI5XiZzuAJ2nRMJTro0voffNltqMo5dFe+Gk9PlL+cJ/6fzpCd6Zx47jvzgvoum15+fsi5d90+kUpp1m9I5dPl2QxsnsCjcKDbMdxKVrozjRuHBjDdzc+DMDOf48HY7TQlXKip79dQ1iQPzf1SbYdxeVoodeAbs+WF3rEww/Atm2W0yjlOX5bn83sDXsZ0y+Z8BB/23FcjhZ6DQivG8jKi0ZSagwHh42AMt3nRanqcpQZnvpmDU3qBzOiW1PbcVySFnoNaT71TV4550Yi5s2m7NVXbcdRyu19uiSLtbvyuHdgSwL9fG3HcUla6DUk0M+XVg/fya+JnSi7+x7YtMl2JKXcVmGJg+d/WEf7JhEMbdfYdhyXpYVeg4a2i2HyqIc4jA9lI0fq1ItSVfTWnC3szCnkwUEtvX4DrhPRQq9BPj7C6OG9GddvND5z5sBLL9mOpJTb2XeoiNd+2cSAVg11A66T0EKvYV2bNSD3ksv4pXlnzAMPwPr1tiMp5Vae/3E9h0sc3D+ope0oLk8LvRY8OCSVBweN4bBvAIwcCQ7dN12pylizM5cPF2QwoltTkqPr2o7j8rTQa0FiZB2GDjyNB/qNhj/+gP/8x3YkpVyeMYbHv1xNWLC/PuJfSVroteSWfsnMSTuTeR16Yx55BFZ7zXnaSlXJD6t388fmfdx5ZgsiQgJsx3ELWui1JCzIn7sGtuTmntdRHFKnfOqlVA/DUOpYikodPPXNGppH1+XyzvG247gNLfRadOnpTYhOjueJQbfAwoXw7LO2Iynlkt6eu5Vt+wp4ZGgqfr5aU5Wlv1O1yNdHeGRoK95v0pn1fQaXb9q1fLntWEq5lOy8Iib8vJH+LaPp1SLKdhy3ooVey7onRTKwdUOu6nQljoh6cNVVUFJiO5ZSLuPZ79ZSWOLgoSGtbEdxO1roFjw8JJX9wWG8ecV9sHQpPPmk7UhKuYRF2w4wfVEm1/RMpFmULlM8VdUudBHxFZElIvKVMwJ5gyb1Q7ipTzJPB6Sw57yLywt98WLbsZSyylFmePTzlTQMC2SMLlOsEmeM0G8D1jjh83iV63s3I75+CNedNgITFVU+9VJUZDuWUtZ8MH8bq3bk8vCQVOoG6umYVVGtQheROGAI8KZz4niPIH9fxp2byrJ8H7697QlYuRIef9x2LKWs2HeoiH9/v47uSQ10N8VqqO4I/UXgXuC42wiKyGgRSReR9Ozs7GpezrP0a9mQAa2iuTs/loIrroJnnilfzqiUl3n2u3UUFDt47NzWuptiNVS50EVkKLDHGLPoRK8zxkw0xqQZY9KionQJ0l89OrQ1pWWGsX2uhtjY8qmXwkLbsZSqNYszDjAtfTujeiTQvGGo7ThurToj9B7AuSKyFfgQ6Cci7zsllReJbxDCTX2SmL4hj5WP/wfWrIFHH7UdS6laUeoo+/NG6G0DWtiO4/aqXOjGmAeMMXHGmARgGPCzMeYKpyXzIjf0TiKhQQhj9kZSet1oeO45+P1327GUqnGT/9jGyiy9Eeosug7dBQT5+/LP89uyZW8+rw4eDU2blu/1UlBgO5pSNWbHwcM8/8M6+qRE6Y1QJ3FKoRtjfjHGDHXG5/JWPZtHckHHWF5auJus5yfAhg3w4IO2YylVY8Z+sYoyY3jivDZ6I9RJdITuQh4a0oo6gX7csac+5uabYfx4+PVX27GUcrrvVu7ix9W7uWNAC5rUD7Edx2NoobuQyLqBPDioFQu27ufjS26BpCQYNQoOHbIdTSmnySssYdwXq2jZKJSreybajuNRtNBdzMVpcXROrM8TszI4+MpE2LoV7rvPdiylnOb5H9azO6+Qpy9si79ujetU+rvpYkSEpy5oQ0FxKY/uqwd33AGvvAIzZ9qOplS1Lck4wOQ/tjKia1M6xtezHcfjaKG7oOToUG7p25wvlu1g5vAxkJICV18Nubm2oylVZUWlDu6dsZxGYUHcMzDFdhyPpIXuom7sk0TLRqE88O1GDr32JmRmwt13246lVJX9d+ZGNuw5xFMXtiU0yN92HI+khe6iAvx8+PdF7dmXX8xj2aFwzz3wxhvw3Xe2oyl1ylZm5fDqr5v4R6c4+qZE247jsbTQXVjbuHCu79WM6Ysy+W34zZCaCtdeCwcP2o6mVKWVOMq4Z8Zy6tcJ4JGhegpRTdJCd3G39m9OcnRd7v9qPfkT34Jdu+D2223HUqrSXv1lE2t25vLP89sQERJgO45H00J3cUH+vjx7UTt25hby5J468MADMHkyfPml7WhKndS6XXn89+cNnNM+hoGtG9mO4/G00N1Ap/h6XNMjkQ/mZzBn2A3Qrh2MHg379tmOptRxFZeWcdf0pYQG+TPunFTbcbyCFrqbuHtgCsnRdbn787XkvfYm7N0Lt95qO5ZSx/XfnzewMiuXpy5oS4O6gbbjeAUtdDcR5O/LC5d0YO+hIh7a6le+Z/oHH8Ann9iOptTfLM44wMuzNvKPTnGc3UanWmqLFrobaRsXzq39yx84+mrQlXDaaXDDDaBH+ykXUlBcyl0fLaNxeDBjz9Wpltqkhe5mbuqTRPsmETz09Tr2TngdcnLgppvAGNvRlALg6W/WsmVvPs9d3J4wfYCoVmmhuxk/Xx9euKQ9RaUO7lztwIwbBzNmwEcf2Y6mFL+uz+a9edu4pmci3ZIa2I7jdbTQ3VCzqLo8OLgVv63P5r0eF0OXLuWj9F27bEdTXmx/fjH3TF9G8+i6uleLJVrobmpE16b0TYninz9sYNOzEyA/v3w+XadelAXGGO6ZvoyDBSW8OKwDQf6+tiN5JS10NyUiPHdxe8KD/bk+PZ/ix56Azz+HKVNsR1Ne6J3ftzJz7R4eHNyS1jHhtuN4LS10N9agbiAvXtqBTdmHGNdsAPToAWPGQFaW7WjKi6zakcPT36xlQKtoruqeYDuOV9NCd3M9kiO5sXcSHyzawaz7/gVFReVPkerUi6oFBcWljJm6hHp1/Hn2ovZ62LNlWuge4I4zW9AxPoJbFx3iwKNPwDffwNtv246lvMC4L1axZW8+L1zagfp1dOMt27TQPYC/rw8vDesIBq6u05my3n3Kj67LyLAdTXmwz5Zk8VF6Jrf0TaZ7UqTtOIpqFLqINBGRWSKyRkRWichtzgymTk2T+iE8e1E7lmTl8tLw+8HhKN87XadeVA1YvzuPBz5ZQefE+tzWv7ntOKpCdUbopcBdxphWQFfgZhHR53wtGtS2Mdf0TOTFTaUsu/1h+PFHmDjRdizlYQ4VlXLD+4uoE+jHhMs64uer/9F3FVX+kzDG7DTGLK54Ow9YA8Q6K5iqmvsHtSStaT0uk/YU9OoLd90FW7bYjqU8hDGG+2YsZ9u+AiZc3pHosCDbkdRRnPJPq4gkAB2B+c74fKrq/H19mHB5J0IC/bi21w0YHx+4+mooK7MdTXmASXO38vWKndwzMIWuzfTRfldT7UIXkbrAx8DtxpjcY/z4aBFJF5H0bN0VsFY0Cg/ipWEdmVdahw+G3QG//AKvvGI7lnJzi7bt5+lv1nBmakOu79XMdhx1DNUqdBHxp7zMpxhjjrkxtzFmojEmzRiTFhUVVZ3LqVPQPTmSu85K4aF6p5PRpTfcdx9s3Gg7lnJTu3IKueH9xcTWC+a5i3W9uauqzioXAd4C1hhj/uO8SMpZbuqTxJB2MVyaNooSP38YObJ89YtSp6CwxMHo99IpKCrljSvTCA/WLXFdVXVG6D2AEUA/EVla8W2wk3IpJxAR/n1xO+o1b8aj/a6DuXNh/HjbsZQbMcZw/8fLWZGVw4vDOtKiYajtSOoE/Kr6E40xcwD9f5eLCwnwY+KVp3FezmGGbvid7g89hAweDC1b2o6m3MDE3zbz2dId3H1WC85MbWg7jjoJXUDqBeLqhfDqiDTu6ncj+b4BmJEjobTUdizl4mat28Mz361lSLvG3Nw32XYcVQla6F6ic2J9xgw/gwf6XY/Mnw/PP287knJh63blcesHS2jVKIx/X9ROb4K6CS10LzK8S1Mir72Sr1N64HjkUVi50nYk5YJ25xYy6u0FBAf48uZVaYQEVHlmVtUyLXQv8/DQ1sy8ZSwH/IPJuXQ4lJTYjqRcSH5RKVe/s5CcwyVMGnk6MRHBtiOpU6CF7mV8fYQnR/fjzcvuJnz1cnY+MA7GjbMdS7mAUkcZt3ywmLW78pgwvBNtYvXkIXcjphZ340tLSzPp6em1dj11fNl5RSztNZi+y3/Fr8yhuzJ6OWMMD3+2kinzM3jygjYM79LUdiR1FBFZZIxJO9nrdITupaJCA0maOokDIWEAHFi6ynIiZdPLszYyZX4GN/RO0jJ3Y1ro3mrcOJq1SiDq0AEA6nVsAyI6/eKF3pu3jed+WM8FHWO5d2CK7TiqGrTQvdW4ceXTLBVTLRkRjSjyD6S4Q0e7uVSt+nxpFo9+vpIBraJ59qJ2+Pjo8kR3poWuAFg941vWNojH78ILKX39ddtxVC2YtXYPd320jM4J9ZlweSf89aAKt6d/ggrGjuXs/h1YPeUzfk3siN8NN2DGjtUbpR5s4db93PD+Ilo2DuXNq9II8ve1HUk5gRa6+nPe/LJ+qax97T0+ajsAefxxzLXX6hYBHmjRtv2MnLSA2HrBTB7VmdAg3T3RU2ihq/9x45mtyHpuAuO7D0MmTcKcdz7k59uOpZxk0bb9XPnWAhqGBTH1uq40qBtoO5JyIi109Td3nJWCY9xjPDjwZsy332L69gU9bcrtLdq2n6smLSwv89FdaajngXocLXR1THcMaE7knWMYfcFDlCxbgeneHTZtsh1LVdGRMo8KDdQy92Ba6OqYRIQ7zmxB6vXDGXbJE+Tv3ovp1g30SV+38/umvVz51oLyMr9Oy9yTaaGr4zpS6n1Hnse5lz7DXuOH6dMHvv3WdjRVST+s2sXItxcSWy+YD0d3pVG4lrkn00JXJyQijOnfnFFXD2TwsGfZWj8Wc8458M47tqOpk5ixKJMbpywmtXEYH13fTUfmXkA3OlaVMqJbAmHB/pwfEMQ7Xz9Lx1GjIDMTHnqofMsA5VLemrOFJ75aTc/kSF4fcRp1AvWvujfQP2VVaed1iCUsqBdXBgbw/PcTOOuRR8pLfcIE8NMvJVdQVmb413dref23zQxu24gXLu1AoJ8+NOQt9G+hOiV9W0bzzvU9uS4oiO116nPN66/Dzp0wdSqEhNiO59UKiku5Y9pSvl+1myu7NWXsOa3x1b1ZvIoWujplpzWtz6c392BUnQAyg+vx6JevI/37w5dfQmSk7XheaXduIddOTmfljhzGnpPKqB6JtiMpC7TQVZU0bVCHT2/swQ2hQdwQUp+Xv34O3x49kO++g0Qtk9q0ekcu10wuPzbujRFpDEhtaDuSsqRaq1xE5GwRWSciG0XkfmeFUu4hPMSfyVeDGVVSAAAKAElEQVR3Jnz4JQy75AkKsnZR1q07LFliO5rX+GxJFhe+Ohdj4KPru2mZe7kqF7qI+AIvA4OAVOAyEUl1VjDlHgL8fPjXP9ox9MaLufDyf7GnqAzHGb3gxx9tR/NoxaVljP18JbdPW0q72Ai+uKWHngGqqjVC7wxsNMZsNsYUAx8C5zknlnInIsLIHok89dAlXHP9eDbUiaJs0GB47z3b0TzSzpzDDJv4B5P/2Ma1PROZcl0XonWNuaJ6hR4LbD/q/cyKjykvdVrT+rzz0AU888DrzItNhSuvpPifT+m+6k40c81uhr40h7W78phweUceHpqqB1OoP1XnK+FY66H+9jdXREaLSLqIpGfrjn0eLyo0kDfH9OeP/77H56m9CXjkIbJHjgaHQ88rrYbDxQ4e/mwF10xOJyo0kC9u6cHQdjG2YykXI6aKoycR6QaMM8YMrHj/AQBjzNPH+zlpaWkmXTd38hrzN2az6epbuHz2R2zseRbJc37Q0XoVrMjM4bZpS9icnc/oXs2466wW+rCQlxGRRcaYtJO9rjoj9IVAcxFJFJEAYBjwRTU+n/IwXZKjGPL9+3xy5d00m1N+kzTj468sp3IfhSUO/vPjei54ZS4FRQ4+uLYLDw5upWWujqvK69CNMaUicgvwPeALTDLGrHJaMuURwv/1JBe++9yf78dfdA4Apddeh98bE23FcnnzNu/jwU9XsDk7n/M7xDDu3NZEhATYjqVcXJWnXKpCp1y8nAhfjriTntPfoF5hHtlDLiBq/L8hKcl2MpdxsKCYp79Zy7T07TSpH8yT57elV4so27GUZbUx5aLUKTvn3edZP3cJ7/UbTp0fvsGRkkLO1aNh1y7b0awqLi1j0pwt9HnuF2YszuSG3kn8cHtvLXN1SrTQVe0ZOxaALp2SuPi7yXw4ZSbTO5xNyORJFCc04/C9D0BOjuWQtcsYw3crd3LWC7/y+FeraRMTzldjenL/oJYEB+hcuTo1OuWirNqTV8i7780kZcK/OGfNbA6HRcCDDxJ82xgI8tyHZYwxzN24j/Ez17Nw6wGaR9flwSGt6NMiCtH95dVf6JSLcgvRoUHcfdMQUmZ9zeNjJ7OwfiLB999LbnwzDr32Rvn69SM8YB27MYafVu/m/Fd+54q35pOxv4AnL2jDt7edQd+UaC1zVS06QlcuZWVWDj+9PJW+7/yH9js3sKdJEo5//pPGIy4FHx+3XcdeWOLgy2U7eGvOFtbuyqNJ/WBu7J3MP06L1WWI6qQqO0LXQlcuaePuXOY88zq93h1Ps/1ZrG/WhhabV1Jy4CD+Ee6zCdWm7ENMmZfBjEXbyS0sJTm6Ljf1SeLc9jH46SP7qpIqW+i6H7pySckNw0h+4R7yAw7As0/TYvNKAPzrRQBwYMAgwj94F5+ovxyoMW6c9amZPbmFfLtyF18v38mCrfvx9xUGtm7EFV2b0iWxvk6rqBqjI3TlFhwlpfgG+DPzrMtoNe8nYnKzKfXxIaNNZxwXXEDcqMsJbhpXfmD1sb6ma7DojTFsys5n9oZsvl25i4Vb92MMtGhYl/M6xHJJWhOiQgNr5NrKO+iUi/I8FWV9qLCERR//SMG06bSc+yOJ+7MoQ9jQrA0pm1ewZtI0Yvt2Jywh7m8/F/j/cj+64B97rNLz844yw5a9h1iScZDfN+3j90172Z1bBJSX+OC2jRnStjHNG4Y65ZetlBa68jzHGGXnF5Zw8OrRxE59528vzw8IZlOr0yho3ZauH7zK8vc+JaxFMxK6tC8v779OfRz1d8FRZsjOK2JHzmF25RSy4+BhNmUfYvWOXNbtzqOwpAyABnUC6JbUgO5JkXRPakBCZB0n/6KV0kJX3kqEJe9+Sv78hSR8MZ247RuO+bLDfoEElxb9z8e6PfUThSUOCkvKKCx1/G3AHhHiT2rjMFo1DiO1cRhtYsNpHl0XHx+dE1c1SwtdeadjzKEXHconMLRupT+FAV74YR3RoYHERATRKCyYmIggwoP99YamskJXuSjvVLG9wNEC61ZMgxwp+iOlf5wpFwHurMGIStUUXQirPMvxVrIco+iV8jQ6Qlfe4eiiP1LuR5f8Y4/VahylaoLOoSullIvTzbmUUsrLaKErpZSH0EJXSikPoYWulFIeQgtdKaU8RK2uchGRbGBbFX96JLDXiXFqmjvl1aw1x53yulNWcK+81c3a1Bhz0hPDa7XQq0NE0iuzbMdVuFNezVpz3CmvO2UF98pbW1l1ykUppTyEFrpSSnkIdyr0ibYDnCJ3yqtZa4475XWnrOBeeWslq9vMoSullDoxdxqhK6WUOgG3KnQRuVhEVolImYi45N1tETlbRNaJyEYRud92nhMRkUkiskdEVtrOcjIi0kREZonImoqvgdtsZzoREQkSkQUisqwir8tv5ygiviKyRES+sp3lZERkq4isEJGlIuLSO/6JSISIzBCRtRVfv91q6lpuVejASuBC4DfbQY5FRHyBl4FBQCpwmYik2k11Qu8AZ9sOUUmlwF3GmFZAV+BmF/+9LQL6GWPaAx2As0Wkq+VMJ3MbsMZ2iFPQ1xjTwQ2WLo4HvjPGtATaU4O/x25V6MaYNcaYdbZznEBnYKMxZrMxphj4EDjPcqbjMsb8Buy3naMyjDE7jTGLK97Oo/wvRazdVMdnyh2qeNe/4pvL3rASkThgCPCm7SyeRETCgF7AWwDGmGJjzMGaup5bFbobiAW2H/V+Ji5cOu5KRBKAjsB8u0lOrGIKYymwB/jRGOPKeV8E7gXKbAepJAP8ICKLRGS07TAn0AzIBt6umM56U0Tq1NTFXK7QReQnEVl5jG8uO9I9yrFOEHbZUZk7EpG6wMfA7caYXNt5TsQY4zDGdADigM4i0sZ2pmMRkaHAHmPMIttZTkEPY0wnyqc3bxaRXrYDHYcf0Al41RjTEcgHauzemssdQWeMGWA7QzVkAk2Oej8O2GEpi8cREX/Ky3yKMeYT23kqyxhzUER+ofx+hSvegO4BnCsig4EgIExE3jfGXGE513EZY3ZUfL9HRD6lfLrTFe+tZQKZR/3vbAY1WOguN0J3cwuB5iKSKCIBwDDgC8uZPIKICOXzkGuMMf+xnedkRCRKRCIq3g4GBgBr7aY6NmPMA8aYOGNMAuVfsz+7cpmLSB0RCT3yNnAWrvkPJcaYXcB2EUmp+FB/YHVNXc+tCl1ELhCRTKAb8LWIfG8709GMMaXALcD3lN+0+8gYs8puquMTkanAH0CKiGSKyDW2M51AD2AE0K9iqdrSihGlq2oMzBKR5ZT/Q/+jMcbllwO6iYbAHBFZBiwAvjbGfGc504mMAaZUfC10AJ6qqQvpk6JKKeUh3GqErpRS6vi00JVSykNooSullIfQQldKKQ+hha6UUh5CC10ppTyEFrpSSnkILXSllPIQ/wdwNq2CAxIhOAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "theta_history = []\n",
    "gradient_descent(0.0, 0.2)\n",
    "plot_theta_history()\n",
    "print(len(theta_history))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XdclvX+x/HXlymyFBmCgKCoOFFza6VpWWppwzTT1Mqm5ek0TvVrWOdU57TXqVOWoywtzbZNV2YucCsOQKYsUQFB9vf3xwWpKYpww3WPz/PxuB+M+/a+3pp9/PC9vkNprRFCCGH7nMwOIIQQwjKkoAshhJ2Qgi6EEHZCCroQQtgJKehCCGEnpKALIYSdkIIuhBB2Qgq6EELYCSnoQghhJ1ya8mL+/v46IiKiKS8phBA2Ly4u7rDWOuB8r2vSgh4REUFsbGxTXlIIIWyeUiqlLq+TIRchhLATUtCFEMJOSEEXQgg7IQVdCCHshBR0IYSwE1LQhRDCTkhBF0IIO2ETBf37HZl8srFO0zCFEMKqZBw7wb9/2EtOYUmjX8smCvrynZm8/NM+SisqzY4ihBAX5LPNabz3WyJlFVWNfi2bKOgT+oZxtLicn3Znmx1FCCHqrLJKsyQ2jYs7BBDasnmjX88mCvqQKH9CW3rw2eZUs6MIIUSd/bY/l8z8Em7qG9Yk17OJgu7kpJjQJ4x1CXmk5BWZHUcIIepk0aZU/L3cGN45qEmuZxMFHeCGPqE4KWM8SgghrF1OQQkr9uZwfe9Q3FyaptTaTEEP9vVgWKdAlsSlU1HZ+DcXhBCiIZZuSaeySjOhiYZbwIYKOsDEfuHkFpaycm+O2VGEEKJWVVWazzan0T/Sj3YBXk12XZsq6MM6BRDo7c5iGXYRQlixDUl5pOQVM7Ff03XnYGMF3cXZifF9Qlm9L4fM/BNmxxFCiLNatDkNn2YuXNUtuEmva1MFHWBCn3CqNCyJTTc7ihBCnOFoURk/7criut6hNHN1btJr21xBD2/VnCFR/ny2OY2qKm12HCGEOM2yrRmUVVY16c3QGjZX0AEm9gsj49gJ1iYcNjuKEEL8SWvN4k2pxIS1oHOwT5Nf3yYL+uVdgmjZ3JXFm2TlqBDCemxJPcqBnONNtjL0r2yyoLu7OHN971B+2ZPN4eOlZscRQggAFm9Ko7mbM2NiQky5vk0WdDCGXSqqNF/Eyc1RIYT5CkvK+W5HJtfEhODl7mJKBpst6FGB3vRp25LPNqehtdwcFUKY6+tthzhRXsnEfuGmZbDZgg7GytGkw0VsSDpidhQhhAPTWrN4cyrRrb2JCfU1Lcd5C7pSaq5SKkcpteuU7/kppX5RSh2o/tiycWOe3ejuwfg0c5HTjIQQptqens+ujAJu6heOUsq0HHXp0OcDV/7le48CK7TWHYAV1V83OQ83Z264KIyfdmeRWyg3R4UQ5li4IYXmbs5c17uNqTnOW9C11r8Bfx3TGAssqP58ATDOwrnq7OYB4ZRXaj6Plf1dhBBN71hxGd9uP8TYnm3wbuZqapb6jqEHaa0zAao/Btb2QqXUHUqpWKVUbG5ubj0vV7v2AV4Mat+KTzemUikrR4UQTWxpXDqlFVVMHmDezdAajX5TVGv9vta6j9a6T0BAQKNcY8qAtmQcO8Eq2VZXCNGEqqo0n2xMpXd4C7qGmHcztEZ9C3q2UioYoPqjqZV0RJcgAr3dWSg3R4UQTeiPxDwOHi5i8oC2ZkcB6l/QvwGmVn8+FfjaMnHqx9XZiYn9wlmzP5fUvGIzowghHMjCDSm0bO7KqO5Nu01ubeoybXERsB7opJRKV0rdBvwbuFwpdQC4vPprU93ULwwnpfhkk3TpQojGl5Vfwi/x2dzYJ6zJt8mtzXnXp2qtb6rlqeEWztIgwb4ejOgcyJLYdP5+eUfcXazjD1gIYZ8WbTImYkzqb/7N0Bo2vVL0ryYPaMuRojJ+2JlldhQhhB0rr6xi8eZULu0YQNtWnmbH+ZNdFfTB7f2JaNWchRtk2EUI0XhWxGeTXVBqNTdDa9hVQXdyUkwe0JbYlKPEZxaYHUcIYac+3pBCiG8zLouudQmOKeyqoAPccFEo7i5O0qULIRpFUu5x1iXkMal/OM5O5u3bcjZ2V9BbNHdjTI8QvtqaQWFJudlxhBB25pONqbg4KW406VSic7G7gg4wZWBbisoq+WprhtlRhBB25ERZJUti0xjZrTWB3s3MjnMGuyzoMaG+dGvjw8cbUuTwCyGExXy7/RAFJRVM7m9dN0Nr2GVBV0pxy8AI9mcfZ31intlxhBB2QGvNvD+S6RTkzYB2fmbHOSu7LOgA18SE4Ofpxrw/ks2OIoSwA5sOHiE+s4BpgyNMPcTiXOy2oDdzdWZSv3B+jc8m7Yjs7yKEaJj5fyTTorkr43qae4jFudhtQQdj5aiTUiyQLl0I0QDpR4v5aXcWE/uG4+FmvduK2HVBb+3bjKu6teaz2DSKSivMjiOEsFEfb0hBKcWUgdZ5M7SGXRd0gOmDIyksqWDZlnSzowghbNCJskoWb0pjZNcg2rTwMDvOOdl9Qe8d3oIeob7M/yOZKjmiTghxgb7cmkH+iXKmDYo0O8p52X1BV0oxfXAEiblF/J5w2Ow4QggborVm/h8H6RriQ9+IlmbHOS+7L+gAo7oH4+/lzrx1B82OIoSwIesT89iffZxpg6x3quKpHKKgu7s4c3P/cFbty+Xg4SKz4wghbMTcdcm08nTj6pgQs6PUiUMUdICbB4Tj6ixTGIUQdZOaV8yKvdlM6h9uNUfMnY/DFPRA72aM6RHCktg02YVRCHFeH61PxlkpbrbSfVvOxmEKOsC0QREUlVWyNE6mMAohaldUWsFnsWlc1T2Y1r7Wt6tibRyqoMeEtaB3eAsWyBRGIcQ5LNuSTmFJBdMHR5gd5YI4VEEHmDY4kuS8YlbuzTE7ihDCClVVGbsqxoT60iushdlxLojDFfSrurUm2LcZH/yeZHYUIYQVWrUvh6TcIm4dEmkTUxVP1aCCrpR6QCm1Wym1Sym1SCll9YNNrs5O3Do4kg1JR9iRfszsOEIIK/P+b0m0aeHBqO7BZke5YPUu6EqpNsD9QB+tdTfAGZhoqWCNaWK/MLzdXZizVhYaCSFO2p52jI0HjzB9cASuzrY3gNHQxC6Ah1LKBWgOHGp4pMbn3cyVm/qHs3xnpuyVLoT405y1SXg3c2Fiv3Czo9RLvQu61joDeBlIBTKBfK31z5YK1timDYpAAfPWJZsdRQhhBdKOFLN8ZyaT+oXj5e5idpx6aciQS0tgLBAJhACeSqnJZ3ndHUqpWKVUbG5ubv2TWlhICw+ujglh8eZU8otloZEQjm7uuoM4KcU0G5uqeKqGDLmMAA5qrXO11uXAMmDQX1+ktX5fa91Ha90nICCgAZezvNsvjqS4rJJPN6WaHUUIYaL84nI+25zGNTEhBPta957n59KQgp4KDFBKNVfG3J7hQLxlYjWNriG+DInyZ966g5RVVJkdRwhhkk82pVBcVsntF7czO0qDNGQMfSOwFNgC7Kx+r/ctlKvJzLikHTmFpXyz3Sbu5wohLKy0opL565K5uIM/XUJ8zI7TIA2a5aK1flprHa217qa1nqK1LrVUsKZySQd/OgV588HaJLSW7QCEcDTfbDtETmEpM2y8OwcHXCn6V0opZlzSjr1Zhaw9ICcaCeFItNbMWZtEdGtvLu7gb3acBnP4gg5wTUwIQT7uzFkr2wEI4UjW7M9lf/ZxZlzczuaW+Z+NFHTAzcWJaYMiWXvgMHsOFZgdRwjRROasTSLIx91mTiQ6Hyno1Sb1D8fTzVm6dCEcxK6MfNYl5DF9cCRuLvZRCu3jd2EBvh6uTOgbzjfbD8l2AEI4gP+tScTL3YWbbHSZ/9lIQT/FjEsicVJIly6EnUvKPc73OzOZPKAtvh6uZsexGCnopwj29eD63qEs3pxGTmGJ2XGEEI3kvTVJuDk7cduQSLOjWJQU9L+489L2VFRWMff3ZLOjCCEawaFjJ1i2NZ0JfcMI8HY3O45FSUH/i0h/T0b3CGHhhhTZtEsIOzRnbRJawx2X2P5Cor+Sgn4W9wxtz/HSCj5an2x2FCGEBeUdL2XRplTG9mxDaMvmZsexOCnoZ9E52Ifh0YHMXXeQ4rIKs+MIISxk3rpkSiuquHuo/XXnIAW9VvcMi+JocTmLNqWZHUUIYQEFJeUsWJ/MlV1bExXobXacRiEFvRYXtW3JgHZ+vP9bIqUVlWbHEUI00MINKRSWVHDvsCizozQaKejncO+wKLILSlm2JcPsKEKIBjhRVsmHaw9yaccAurXxNTtOo5GCfg5DovzpEerL/9YkUlEpB2AIYas+25xKXlGZXXfnIAX9nJRS3DM0ipS8Yr7fmWl2HCFEPZRVVPH+b0n0jWhJv0g/s+M0Kino53FFlyCiAr14Z1UiVVVyAIYQtuarbRkcyi/hHjvvzkEK+nk5OSnuGdqefdmFrNibY3YcIcQFqKzS/G91Il2CfRja0boOqW8MUtDr4OqYENq2as4bK/bLMXVC2JBvtmeQdLiI+y6LsosDLM5HCnoduDo7MXNYFLsyCvhlT7bZcYQQdVBRWcWbKxKIbu3NyK6tzY7TJKSg19G1vdoQ0ao5r/96QLp0IWzA19sOcfBwEX8b0REnJ/vvzkEKep25ODtx32Ud2JNZwE+7pUsXwppVVFbx1soDdAn2YWTXILPjNBkp6BdgbM8QIv09ef3X/TLjRQgr9tW2QyTnFfO3ER0cYuy8RoMKulKqhVJqqVJqr1IqXik10FLBrJHRpUexN6uQn3ZnmR1HCHEWNd151xAfLu/iON05NLxDfwP4UWsdDcQA8Q2PZN2uiQmhnb8nr/96QLp0IazQsq0ZpOQV87cRHR2qO4cGFHSllA9wCfAhgNa6TGt9zFLBrJWLsxP3D+/AvuxCftglXboQ1qS8ujvv3saXEZ0DzY7T5BrSobcDcoF5SqmtSqkPlFKeFspl1a6OCaF9gCdvrJCxdCGsybIt6aQdOeFwY+c1GlLQXYDewLta615AEfDoX1+klLpDKRWrlIrNzc1twOWsh7OT4v7hHdiffVz2eBHCSpRVVPHWygRiQn25LNrxunNoWEFPB9K11hurv16KUeBPo7V+X2vdR2vdJyDAfpbejukRQlSgF2+sOECldOlCmO6LLemkHz3hkGPnNepd0LXWWUCaUqpT9beGA3ssksoGODspZg3vQELOcb7bccjsOEI4tLKKKt5emUDPsBYM7WQ/jeOFaugsl/uAT5RSO4CewPMNj2Q7RncPpmOQ0aXLfulCmGdJXBoZxxx37LxGgwq61npb9XBKD631OK31UUsFswVOTooHRnQkKbeIZVvlVCMhzFBSXsmbKw5wUduWXOoAOyqei6wUbaAru7WmR6gvb/x6QM4eFcIEC/5IJruglEdGdnLo7hykoDeYUopHRkaTcewEn2xINTuOEA4l/0Q576xOZGinAPq3a2V2HNNJQbeAIR38GdS+Ff9dlcDx0gqz4wjhMOb8lkT+iXIeuqLT+V/sAKSgW8jDIzuRV1TG3N8Pmh1FCIeQW1jK3HUHGdMjmG5tfM2OYxWkoFtIr/CWXNEliPd/S+JIURnMnm12JCHs2n9XJVBaUcWD0p3/SQq6BT00shPFZRW8uzoBnnnG7DhC2K20I8V8sjGFG/uEEenvEDuO1IkUdAvqGOTNlEgPLps5yfhGYaG5gYSwU6/9uh8nZSzuEydJQbek2bN55s7hDEzebnzt4wNKwS23QKVMaRTCEvYcKuDLrRlMGxRBa99mZsexKlLQLWn2bNCa577dBcDRydPA1xc+/hgiIuDxx2HfPjMTCmHzXvghHp9mrtwzNMrsKFZHCnojmHlZRwBmDbsbMjNh8WLo3h3+8x+IjoYBA+Ddd+GoQy2sFaLBftufy9oDh7nvsih8m7uaHcfqSEFvBL7NXdky7X7jL1/6cZgwAZYvh/R0eOklKCqCe+6B1q1h/Hj47jsoLzc7thBWrbJK8/zyeML8PJgysK3ZcaySFPRG0nXOq4T5efD88r0nt9cNDoaHHoIdOyAuDu66C1avhquvhtBQ+PvfYft2U3MLYa2+3JrB3qxCHhkZjbuLs9lxrJIU9Ebi7uLMwyOjic80buCcRino3RveeAMyMuCrr2DwYHj7bejZ03i89hpkZ5sTXggrU1JeySs/7yMmrAVjegSbHcdqSUFvRGO6BxMT6ssrP++jpLyWWS5ubjB2LCxbZoy3v/UWuLoa3XqbNkb3vnQplJY2bXghrMiHvx8kM7+Ex6+KdvgNuM5FCnojcnJSPDaqM5n5JXxYly0BWrWCmTNh82bYvRsefBC2bDHG2YODjXH3jRtBywlJwnHkHS/lf6sTGdE5SDbgOg8p6I1sQLtWXN4liHdWJZBTUFL3X9ilizErJjUVfvwRrrwS5s0zZsh06QIvvGDcZBXCzr3yy35OlFfy6FXRZkexelLQm8DjozpTVlnFyz/XYw66szOMHAmffgpZWTBnDvj7G3Paw8Ph8sth4UJj5owQdiY+s4DFm1KZMrAtUYFeZsexelLQm0CkvyfTB0eyJC6dXRn59X8jX1+4/XZYuxYSEuDJJ42PU6YYUyBvvRXWrIEqOQ5P2D6tNc9+uwcfD1dZ4l9HUtCbyMzLovBr7sYz3+5GW2IMvH17YwOwxERj6uP48bBkCQwdajz39NPGc0LYqJ/3ZLM+KY+/X96RFs3dzI5jE6SgNxGfZq48eEUnNicfZfnOLMu9sZMTXHopzJ1rDMl8/DFERcE//2l8vPhi+OADyG/ATwZCNLHSikqeXx5Ph0AvJvULNzuOzZCC3oQm9A2jc7APzy+Pr30aY0N4esLkyfDLL5CSAs8/D7m5MGOGMSQzaRL89JNsFCas3rx1yaTkFfPkmC64OEuZqiv5k2pCzk6KJ8d0JuPYCT5Ym9S4FwsLg8ceg/h42LABpk8/OVsmPBz+8Q/Ys6dxMwhRD7mFpby9MoHh0YFc0jHA7Dg2RQp6ExvU3p+RXYP476pEMvNPNP4FlYL+/eGdd4yFS0uWGKtUX3kFunaFvn2NFap5eY2fRYg6ePHHvZSUV/J/ozubHcXmSEE3wROju1ClNf/6Pr5pL+zuDjfcAN9+a2w58OqrxqZg991nLFy67jr4+mvZKEyYJi7lKEvi0rltSCTtAmSa4oVqcEFXSjkrpbYqpb6zRCBHEObXnHuGRvH9jkzWJRw2J0RQEDzwAGzbZjxmzoR162DcOAgJgVmzjFWqsipVNJHKKs1TX+8iyMed+2SaYr1YokOfBTRxq2n77ry0HeF+zXnq612UVZg8bzwmxujW09ON7n3oUPjf/+Cii6BHD3j5ZWO4RohG9OnGFHYfKuCJ0V3wcncxO45NalBBV0qFAqOBDywTx3E0c3Vm9jVdSMwtYt66Ouzz0hRcXWHMGGOcPTPTGHf39ISHHza29x01Cj77DEouYAsDIeog73gpL/20j0HtW8luig3Q0A79deARoNYWUyl1h1IqVikVm5ub28DL2ZfLooMY0TmQN1YcICvfyoqknx/cfbcxQyY+3pgVs3MnTJxoTIG8805Yv16GZIRFvPjjPorLKnnmmq6ym2ID1LugK6XGADla67hzvU5r/b7Wuo/Wuk9AgExB+qunxnSlokrz3HIrHrWKjjbmtCcnG3Pcr77aWMA0aBB06gTPPWdsIiZEPWxJPcpnsWlMHxxBhyBvs+PYtIZ06IOBa5RSycBi4DKl1EKLpHIg4a2ac8/Q9ny7/ZB5N0jrytkZRowwinl2trE6NSQEnnjCOAR7+HD46CM4ftzspMJGVFRW/XkjdNaIjmbHsXn1Luha68e01qFa6whgIrBSaz3ZYskcyF2XtieiVXOe+GpX46wgbQze3sZipdWrISkJZs82OvipU40hmalTYeVK2ShMnNOC9SnsypAboZYi89CtQDNXZ/41rjsHDxfxzqoEs+NcuMhIeOopY+fHtWvhppuMY/WGDzeee+IJOHDA7JTCyhw6doJXft7H0E4BciPUQixS0LXWq7XWYyzxXo5qSAd/ru3VhnfXJJKQU2h2nPpRCoYMMfZsz8oy9nDv3Nk4jKNjR+Pc1Pffh2PHzE4qrMDT3+ymSmv+Obab3Ai1EOnQrcj/je6Mp7sLjy/bRVWVjc8e8fAwOvUffzRumP7nP0Yhv/NOY0hm4kT44QeoqDA7qTDBj7uy+GVPNg+M6EiYX3Oz49gNKehWxN/Lncev6sym5CMsiUszO47ltGkDjzwCu3YZ56XOmGHMlhk1ythE7OGHjeeEQygsKWf2N7uJbu3NrUMizY5jV6SgW5nxfULpF+nH88v3cvh4qdlxLEsp6NMH3nrLWLi0bJmxcdjrr0P37sbK1DffNLb8FXbrlZ/3k11YwgvXdcdVtsa1KPnTtDJKKZ6/thvFZRU8+60db2/r5gbXXmvcPD10CN54w/j+rFnGVMhx4+DLL6GszNycwqK2ph5lwfpkpgxoS6/wlmbHsTtS0K1QVKA3M4d14Jvth/hlT7bZcRpfQADcfz/ExcGOHfC3v8HGjcbujyEhxm6QsbGyKtXGlVZU8sjSHbT2acbDIzuZHccuSUG3UncPbU90a2/+78ud5Bc70Ha23bvDSy9BWhosX24sZJozx9i3vVs3ePFFo6MXNuetFQkcyDnO89d1x7uZq9lx7JIUdCvl5uLESzfEkFdUxr++t+Ohl9q4uMBVV8HixcYUyPfegxYtjD1lwsKMk5cWLYITTXBIiGiwXRn5vLsmket7hzKsU6DZceyWFHQr1j3UlzsvaceSuHTW7HfgG4UtWsAddxj7te/fD48/bmwYNmmSMQVyxgz4/XcZkrFS5ZVVPLx0B36ebjw5Rk4hakxS0K3c/cM7EBXoxWNf7KCwxIGGXmrToQP8859w8KCxtcC11xqd+sUXG889+6yxBYGwGu+uTiQ+s4B/jetGi+ZuZsexa1LQrVwzV2devKEHmQUlvPDDXrPjWA8nJxg2DObPN4Zk5s83Dr9++mlju4GhQ2HePCi00VW3dmJfViFvrTzA1TEhjOza2uw4dk8Kug3oHd6S2wZH8unGVNYecOChl9p4eZ3cDCw52ejgDx2CW281hmSmTIFff4VKG9n4zE6UVVTx4JJteDdzZfbVXcyO4xCkoNuIh0Z2IirQi4eX7HCsWS8Xqm1bYzOwffvgjz+MYv7tt3D55cYWv48/bjwnGt1bKw+wK6OA56/tTisvd7PjOAQp6Daimaszr93Yk8PHS3nya1kmf15KwcCBxtmoWVnG0Xk9ehh7ykRHw4AB8O67cOSI2Unt0pbUo/x3VQLX9w7lym4y1NJUpKDbkO6hvtw/3Fhw9O12mYtdZ82awY03wvffQ0aGceh1URHccw8EB8P48fDdd1AuP/lYQnFZBQ9+vp1gXw+evkaGWpqSFHQbc8/Q9sSEteCJr3aRXWBl55Dagtat4cEHjRWpcXFw113GIR1XX20chP33v8P27WantGkvLN/LwcNFvDw+Bh9ZQNSkpKDbGBdnJ167MYbSikoeXroDLXOv60cp6N3b2EMmI8PYU2bwYHj7bejZ03i89ppx1J6oszX7c/l4Qwq3DYlkYPtWZsdxOFLQbVC7AC8eH9WZ3/bn8tH6FLPj2D43Nxg71tj9MTPT2A3S1dXo1tu0Mbr3pUuh1M52v7SwI0VlPLxkOx0CvWSvFpNIQbdRUwa0ZVinAJ5bHk98ZoHZcexHq1Ywc6axb/vu3cbwzJYtxjh7cLAx7r5xo6xK/QutNQ8v2c6x4nJen9iTZq7OZkdySFLQbZRSipfHx+Dr4cp9i7ZSXCYn/1hcly7GrJjUVOPkpSuvNBYrDRhgPPfCC5CebnZKqzD/j2RW7M3h8VHRdA3xNTuOw5KCbsNaebnz+oSeJOYe55/fOeAGXk3F2RlGjjTOSM3KMnZ/9Pc35rSHhxtz3BcuNGbOOKDdh/J5YfleRnQOZOqgCLPjODQp6DZucJQ/d1/ankWb0vh+R6bZceyfry/cfjusXQsJCfDkk8bHKVOMGTS33gpr1kBVldlJm0RxWQX3LdpKS09XXrwhRg57NpkUdDvwwOUd6RXegkeX7SDtSLHZcRxH+/bwzDOQmGhMfRw/HpYsMfaRad/e2FcmMdHslI1q9je7OXi4iNcm9MTPUzbeMpsUdDvg6uzEmxN7gYb7Fm2lrMIxukOr4eQEl14Kc+caQzIffwxRUcaeMlFRxk6QH3wA+flmJ7Wor7Zm8HlsOjOHRTGovb/ZcQQNKOhKqTCl1CqlVLxSardSapYlg4kLE+bXnBdv6MG2tGM8vzze7DiOy9MTJk+GX36BlBR4/nnj0OsZM4whmUmT4KefbH6jsP3ZhTy2bCf9Iv2YNbyD2XFEtYZ06BXAg1rrzsAA4F6llKzzNdFV3YO5bUgk8/9I5hvZGsB8YWHw2GPGYRwbNsD06Sdny4SHG6cv7bG9m9nHSyu4a2Ecnu4uvH1TL1yc5Qd9a1Hv/xJa60yt9ZbqzwuBeKCNpYKJ+nn0qmj6tG3Jo1/s4EC27AVuFZSC/v3hnXeMhUtLlhirVF95Bbp2Nc5LffttyMszO+l5aa35x9IdpOQV8/akXgT6NDM7kjiFRf5pVUpFAL2AjZZ4P1F/rs5OvD2pN83dnLn7ky0Ulcr8dKvi7g433GBs6ZuRAa++amwKdt99xsKl666Dr7+22o3C5q5L5vudmTw8shMD2snSfmvT4IKulPICvgD+prU+Y8miUuoOpVSsUio2N1cOZ2gKrX2b8ebEXiTlHufRZTtlvxdrFRQEDzwA27YZj5kzjXNTx42DkBCYNctYpWol//3iUo7wwvJ4Lu8SxJ2XtDM7jjiLBhV0pZQrRjH/RGu97Gyv0Vq/r7Xuo7XuExAQ0JDLiQswKMqfB6/oxLfbDzFnbZLZccT5xMQY3Xp6utG9Dx1q7OV+0UXGPu4vv2wM15gkK7+EuxZuoU1LD14eL/PNrVVDZrko4EMgXmv9quUiCUu5Z2h7RncP5t8/7GX1vhyz44i6cHWFMWOMcfbMTGPc3dMTHn7Y2N531CjjsI6Spts6uaS8kjvn1Z8NAAASUUlEQVQ+jqW4tII5t/TB10O2xLVWDenQBwNTgMuUUtuqH6MslEtYgFKKl8b3ILq1D/ct2kpi7nGzI4kL4ecHd99tzJCJjzdmxezcCRMnGlMg77zTOGavEYdktNY8+sUOdmbk8/rEXnQM8m60a4mGU005vtqnTx8dGxvbZNcThvSjxYx9ex2+zV356t7BcuiALaushFWrYMEC+OILOHECOnSAW24xHuHhFr3ce2sSeeGHvTx0RUdmXibzzc2ilIrTWvc53+tkAqkDCG3ZnHcnX0RqXjH3L9pKZZV13GQT9eDsDCNGGKtRs7ON1akhIcaeMhERMHw4fPQRHG/4T2Or9uXw7x/3MrpHMPcOi2p4dtHopKA7iH6Rfjwztiur9+Xy3PeyktQueHsbi5VWr4akJJg9G5KTYepUY0hm6lRYubJeG4Xtyyrk/k+30rm1Dy/d0ENugtoIKegO5Ob+bZk+OIK56w4yf91Bs+MIS4qMhKeeMnZ+XLsWbrrJOFZv+HDjuSeegAMH6vRW2QUlTJ+3CQ83Zz6Y2ofmbi6NHF5YihR0B/PE6C5c0SWIZ77bw8+7s8yOIyxNKRgyxNizPSvL2MO9c2fjMI6OHWHQIHjvPTh27Ky/vKi0glvnbyb/RDlzp/UlpIVHE/8GRENIQXcwzk6KNyb2okcbX+5fvJXtaWf/H1vYAQ8Po1P/8UdISzNOX8rPh7vuMoZkJkyAH36ACmM1cdVTTzPz0y3szSrk7Zt7062NnDxka6SgOyDjR+m+BHi7c9uCzbKHuiMICYFHHoFdu4zzUmfMgF9/Nea1h4WhH3oIp38+y6p9uTw7tivDOgWanVjUgxR0BxXg7c68af0oq6hi6rxN5B2XE+0dglLQpw88+6wxQ6ZnT8jKQr3yCgAPdvfh5v5tTQ4p6ksKugOLCvTiw2l9OXTsBFPnbaKwxDo3hBINVFVldOZz5hhH5HXubCxaGjfO2EPmFPdNvsQo+rNnm5NVNIgsLBKs2pvDjI9iuahtSxbc2o9mrs5mRxINceyYsbp0/XrjsXEjFFTvm9eqFQwc+OfjO/c23PftAYZHB/LBtH5WsxGYOF1dFxZJQRcAfL0tg799to3h0YG8O/kiXOXQAttQVWVsC1BTvNevN74G42i87t1PK+BERRkdOGf5h9zNRQq6laprQZcJpgKAsT3bUFBSwZNf7eLhJdt59caeODnJYhKrc+yY0XGf2n3XnFXq52cU7ZtvNj727WssPjqLzclHuGthHNHB3nwwtY/xU9nTTzfhb0Q0Bino4k9TBrSl4EQ5L/20D1dnJ/5zfQ8p6maqqoK9e8/svrU2uu9u3YyNumq67w4d/uy+zyUu5QjT5m6iTUsPFkzvh3fN3j4ybm7zpKCL09w7LIqyiireWGGsKpSi3oTy88/svmsWAPn5wYABxrzygQOhX79au+9ziUs5wi0fbiLIpxmLZgyglZe7hX8TwkxS0MUZHri8Ixp4c8UBlIJ/XydF3eKqqmDfvtO77z17jO5bKaP7vvHGk913x4516r7PJS7lCFPnbjaK+R0DCJLzQO2OFHRxVg+M6ABa8+bKBECKeoMVFJzZfR89ajzXsqXRfU+YcLL79vGx6OVrinmAt7sUczsmBV2clVKKBy7vCMCbKxOoqNT854YeMvulLrQ+s/vevftk9921q3FQ9Kndt1Pj/bn+kXiYGQtiCaweZpFibr+koIta1RR1V2cnXvllPwUl5bw9qbfMU/+rggLYtOlk8d6w4WT33aKF0X2PH3+y+/Ztuj1Sft6dxcxFW4lo1ZyPb+svxdzOSUEX56SU4r7hHWjR3JWnvtnN1Lmb+GBqn5MzIxyN1rB//+nd965dJ7vvLl3g+utPdt+dOjVq930uS+PS+ccXO+jexpf50/vSormbKTlE05GCLupkysAIfDxcefDz7Uyas5H50/s6xgyJwsIzu+8jR4znfH2N7rumgPfrZ3TkVuDD3w/yz+/2MCTKn/emXISnu/yv7gjkv7Kos7E92+DTzJW7P4njhv+tZ+60vkT6e5ody3K0Ng6B+Gv3XXPiT5cucO21J7vv6GjTuu/aVFVp/vPjXt77LYlR3Vvz2oSeuLvIEJmjkKX/4oLFpRxhxkdxVGnNe5Mvon+7VmZHqp/jx8/svvPyjOd8fIzuu6Z49+9vNd13bYrLKnjgs238tDubWwa25emru+IsM5PsguzlIhpVSl4R0+cbe6n/+7oeXH9RqNmRzk1r43i2U7vvnTtPdt+dO5++50nnzlbXfZ9LdkEJty+IZdehfJ4a04XpgyPNjiQsSPZyEY2qbStPvrx7MHctjOPBJdtJzivigREdrWeu+vHjxkEOp3bfhw8bz/n4GB33E0+c7L5btjQ3bwPsOVTAbQuMY+PmTOnDiC5BZkcSJmlQQVdKXQm8ATgDH2it/22RVMIm+DZ3ZcGt/Xjiq528tTKB+MwCXrmxJ74eTTwDRmtITDyz+66sNJ6Pjoarrz69+3a2j3Hlr7Zm8OiyHbTwcOPzOwfKsXEOrt5DLkopZ2A/cDmQDmwGbtJa76nt18iQi33SWrPgj2T+9X08bVp68O7NF9ElxLIrHU9TVHRm952bazzn7W103KeOffv5NV4Wk5RVVPHc93tYsD6FfhF+vD2pF4Eyx9xuNcWQSz8gQWudVH3BxcBYoNaCLuyTUoppgyPpHurLPZ9s4bp31/HcuO6WGVfXGpKSTu++d+w42X136gSjR58s4F262E33XZvM/BPc+8kWtqQe4/YhkfzjqmhZwSuAhhX0NkDaKV+nA/0bFkfYsova+vHdfRcz89MtPLhkO5sOHuGpq7tc2Bzo4uIzu++cHOM5Ly+j437ssZPddysbnWFTTyvis3lk6Q5OlFfy9qRejOkRYnYkYUUaUtDPdvfrjPEbpdQdwB0A4eHhDbicsAUB3u58cnt/Xv1lP++uSWTjwTxem9CTXuEtjf22T91zW2s4ePD07nv79pPdd8eOcNVVJ6cPdutm9913bU6UVfLc8j0s3JBKdGtv3p7Ui6jAC98+V9i3hoyhDwRma61HVn/9GIDW+oXafo2MoTuWjUl5/P3z7WQVlHD/ZR2YdXlHWLPm9O47O9t4sZeXsdKyZuhkwACH675rszM9n1mfbSUpt4g7LmnHg1d0lMVCDqYpxtA3Ax2UUpFABjARmNSA9xN2pn+7ViyfdTFPf72L3JdeN7556aUnX9CiBcyYAZMnw+DBDtt916akvJJ3VifyzqoE/L3c+fT2/gyK8jc7lrBiDVpYpJQaBbyOMW1xrtb6uXO9Xjp0BzR7NjzzzPlf5+cH7dtDu3YnHzVfh4Y6XLHfkJTH41/uJCm3iHE9Q5h9TVfZXMuByUpRYX2U4qHPt7E0Lp0uzat4LsaTXhVHjVksSUnGXPKkJEhJgYqKk7/O1RXatj29yJ/6sPBhEGY6VlzGC8v38llsGmF+Hjw3rjuXdAwwO5YwmRR0YX2UAq35I+Ewj3+5k+S8YkZ2DeIfV0bTLsDr5OsqKiA9/fQif+qjZrfDGv7+Zxb5msLfpo1NdPdlFVUs3JDCmysPUFhSwYyL2zFreAc83Kw/u2h8UtCF9TlllktJeSUfrE3i3dWJlFZUMXlAW2YN70BLzzoMKxw7dmaRP7W7r5klA0Z3HxFx9s6+Xbt6HbRsSVprftqdxb9/2EtyXjFDovz5v9Gd6RxsPz91iIaTgi5sQk5hCa//eoDFm1LxdHfhzkvaMWVgRP23D6iogLS0Mwt9zaPmJKEaAQFn7+xruvtG2qBLa826hDzeWLGfzclH6RDoxeOjOzO0YwCqgYdBC/sjBV3YlP3Zhfznh72s2JuDt7sLUwdFcOuQSPzq0rFfiKNHz+zuawp/aurp3b2b29m7+/btITLSmGpZV9U/nWitWRGfw1urEtiedowgH3fuH96BCX3CcJHVnqIWUtCFTdqVkc87qxP4YVcWzVycmdgvjMkD2tI+4AKKZ32Vl9fe3ScmQn7+6a8PDDx7Z9+uHYSEnN7dK8WSzal8+PtB9mYVEubnwd2XRnH9RW1kTrk4LynowqYl5BTyzqpEvt1xiPJKzaD2rbi5f1uu6Bpk3r4lR4+e/SZtzdh9zd7qAO7uEBlJUWg4Oz0CGfDtQiL+8R1RgV7cM7Q918SESEcu6kwKurALuYWlfB6bxqcbU8k4dgJ/L3fG9AhmVPdg+rRteeb+63/dXqC+KiuN04sOHzZ2cqz5eOrnp37MyYGysrq999NPWyajcBhS0IVdqazS/LY/l882p7FqXw6lFVUEerszqnswl0UH0jfCz5jiVz018gxFRWcvxLUV6aNHz/4+YMx7DwgwpksGBKD9/Tnm6UuC9mDzcSfiil040swHn/BghgzozLhLOxPg61H7+wlxHnJikbArzk6KYdGBDIsO5HhpBSvis1m+M5NFm1KZ/0cybs5OjHE9xqtA7rjxtCjKx/VI3skifeLE2d9YKQgKgqws8PWFkSONIZTRo/8s2H9+jImB0lIqXVw5ePg4W1OP8UdiHn8kHia7oBSAjlFejOoezOjuwXQIks2zRNOSDl3YtKLSCnIffIyI/75s0ffVbm5oV1eqnF2odHYhp2NXwjb+xti31rIvu5CScmO8vJWnGwPbt2JQe38GtW9FhL/n2d/QUkNBwiHJkItwTEqxZutBkg5mk5ySzaG0HI7mHKV56Qk8y05w56Yv6Jl5oMGX2TPjAZyffYYOgV7Wc46qsFsy5CIc1qU9I7i0Z8SfX5dWVFLy+FP4vnThR96m3jKDEw88TOtWXviGh/w5Dt7FUmGFsCAp6MK+PP30Gd9yd3HG/cXn4MXqzUBPvXFa83nN6syaz6uflyNZhC2RibDCvjTmOPVZ/rEQwppIQReO59TCfOrnNYdv1Fa45aamsHJyU1QIIaxcXW+KSocuhBB2Qgq6EELYCSnoQghhJ6SgCyGEnZCCLoQQdqJJZ7kopXKBlHr+cn/gsAXjNDZbyitZG48t5bWlrGBbeRuata3WOuB8L2rSgt4QSqnYukzbsRa2lFeyNh5bymtLWcG28jZVVhlyEUIIOyEFXQgh7IQtFfT3zQ5wgWwpr2RtPLaU15aygm3lbZKsNjOGLoQQ4txsqUMXQghxDjZV0JVS45VSu5VSVUopq7y7rZS6Uim1TymVoJR61Ow856KUmquUylFK7TI7y/kopcKUUquUUvHVfwdmmZ3pXJRSzZRSm5RS26vzPmN2pvNRSjkrpbYqpb4zO8v5KKWSlVI7lVLblFJWveOfUqqFUmqpUmpv9d/fgY11LZsq6MAu4DrgN7ODnI1Syhn4L3AVxqE2NymlrPlwm/nAlWaHqKMK4EGtdWdgAHCvlf/ZlgKXaa1jgJ7AlUqpASZnOp9ZQLzZIS7AMK11TxuYuvgG8KPWOhqIoRH/jG2qoGut47XW+8zOcQ79gAStdZLWugxYDIw1OVOttNa/AUfMzlEXWutMrfWW6s8LMf6naGNuqtppw/HqL12rH1Z7w0opFQqMBj4wO4s9UUr5AJcAHwJorcu01sca63o2VdBtQBsg7ZSv07HiomOrlFIRQC9go7lJzq16CGMbkAP8orW25ryvA48AVWYHqSMN/KyUilNK3WF2mHNoB+QC86qHsz5QSnk21sWsrqArpX5VSu06y8NqO91TnO34d6vtymyRUsoL+AL4m9a6wOw856K1rtRa9wRCgX5KqW5mZzobpdQYIEdrHWd2lgswWGvdG2N4816l1CVmB6qFC9AbeFdr3QsoAhrt3prVHRKttR5hdoYGSAfCTvk6FDhkUha7o5RyxSjmn2itl5mdp6601seUUqsx7ldY4w3owcA1SqlRQDPARym1UGs92eRctdJaH6r+mKOU+hJjuNMa762lA+mn/HS2lEYs6FbXodu4zUAHpVSkUsoNmAh8Y3Imu6CUUhjjkPFa61fNznM+SqkApVSL6s89gBHAXnNTnZ3W+jGtdajWOgLj7+xKay7mSilPpZR3zefAFVjnP5RorbOANKVUp+pvDQf2NNb1bKqgK6WuVUqlAwOB75VSP5md6VRa6wpgJvATxk27z7XWu81NVTul1CJgPdBJKZWulLrN7EznMBiYAlxWPVVtW3VHaa2CgVVKqR0Y/9D/orW2+umANiII+F0ptR3YBHyvtf7R5Eznch/wSfXfhZ7A8411IVkpKoQQdsKmOnQhhBC1k4IuhBB2Qgq6EELYCSnoQghhJ6SgCyGEnZCCLoQQdkIKuhBC2Akp6EIIYSf+H2Hr0bZtgKikAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "theta_history = []\n",
    "gradient_descent(0.0, 0.8)\n",
    "plot_theta_history()\n",
    "print(len(theta_history))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "OverflowError",
     "evalue": "(34, 'Result too large')",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOverflowError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-c10a1af42184>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m# 如果指定学习率为超过1的数，会导致程序最后的数字过大，造成栈溢出的问题\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mtheta_history\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mgradient_descent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0.0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1.1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mplot_theta_history\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtheta_history\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-10-b231f34aa899>\u001b[0m in \u001b[0;36mgradient_descent\u001b[1;34m(initial_theta, eta, epsilon)\u001b[0m\n\u001b[0;32m      9\u001b[0m         \u001b[0mtheta\u001b[0m \u001b[1;33m-=\u001b[0m \u001b[0meta\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[0mtheta_history\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtheta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m         \u001b[1;32mif\u001b[0m \u001b[0mabs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mJ\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtheta\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mJ\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlast_theta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m             \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-6-bd41aa589cfc>\u001b[0m in \u001b[0;36mJ\u001b[1;34m(theta)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mJ\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtheta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtheta\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m2.5\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m**\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m-\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mOverflowError\u001b[0m: (34, 'Result too large')"
     ]
    }
   ],
   "source": [
    "# 如果指定学习率为超过1的数，会导致程序最后的数字过大，造成栈溢出的问题\n",
    "theta_history = []\n",
    "gradient_descent(0.0, 1.1)\n",
    "plot_theta_history()\n",
    "print(len(theta_history))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 为了程序在使用的过程中更友好，修改J(theta)方法，加入异常处理，\n",
    "# 在数字超过一定限度的时候，直接返回一个无穷大\n",
    "def J(theta):\n",
    "    try:\n",
    "        return (theta - 2.5) ** 2 - 1\n",
    "    except:\n",
    "        return float('inf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 修改了J(theta)方法后，\n",
    "# 相应地用梯度下降法求解最小值的方法中的循环退出条件也需要增加判断条件\n",
    "# 如果不加退出条件，会导致J(theta)和J(last_theta)都为无穷大，作差后不会小于epsilon，造成死循环的问题\n",
    "def gradient_descent(initial_theta, eta, n_iters=1e4, epsilon=1e-8):\n",
    "    theta = initial_theta\n",
    "    theta_history.append(theta)\n",
    "    i_iter = 0\n",
    "    while i_iter < n_iters:\n",
    "        gradient = dJ(theta)\n",
    "        last_theta = theta\n",
    "        \n",
    "        theta -= eta * gradient\n",
    "        theta_history.append(theta)\n",
    "        if abs(J(theta) - J(last_theta)) < epsilon:\n",
    "            break\n",
    "        i_iter += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta_history = []\n",
    "gradient_descent(0.0, 1.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10001"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(theta_history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_history[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XlwXNWdL/DvT6u1L5Ysy1q8G1vyIi9gCARMQkgCYSALWcjLEELi/BHyXqpS9bLMe2XzpqYek4UUeWRScd5QgamEJQkEBhKWZEJ4A4GAjQN4l/EmS5YlW7IlS9bWv/fHr29ud9/uVqulVm/fT9Wt27r3tnwaoe+5Oufcc0RVQUREmSsn2QUgIqLEYtATEWU4Bj0RUYZj0BMRZTgGPRFRhmPQExFluEmDXkSaROSPIrJPRPaIyH/zH98uIidFZLd/uyHgPd8SkXYROSAiH0zkByAiouhksnH0IlIPoF5Vd4lIGYCdAG4B8EkAg6r6vZDrWwA8DOAyAAsA/B7AClWdSED5iYhoEpPe0atql6ru8r8eALAPQEOUt9wM4BFVHVHVIwDaYaFPRERJkDeVi0VkEYD1AF4DcCWAu0Tk7wG8AeDrqtoHqwReDXhbB6JXDKipqdFFixZNpShERFlv586dvapaO9l1MQe9iJQC+DWAr6nqeRH5MYB/BKD+/fcBfAGAhHm7p31IRLYC2AoAzc3NeOONN2ItChERARCRY7FcF9OoGxHJh4X8z1X1cQBQ1W5VnVBVH4Cfwm2e6QDQFPD2RgCdod9TVXeo6iZV3VRbO2mFREREcYpl1I0A+FcA+1T13oDj9QGXfRTAO/7XTwH4tIgUishiAMsB/GXmikxERFMRS9PNlQA+B+BtEdntP/ZtAJ8RkTZYs8xRAF8GAFXdIyKPAdgLYBzAVzjihogoeSYNelX9T4Rvd/9tlPf8E4B/mka5iIhohvDJWCKiDJf+Qb99e7JLQESU0tI/6O++O9klICJKaekd9AcP2v7JJ4GODoDLIhIReUzpydiUsX178J38LbfYvrgYuOYaYMMGYONG25qaAAnXl0xElB0mndRsNmzatEnjfjJWBHjlFWDnTmDXLtvv2QNM+Ed01tRY4AeG/8KFDH8iSnsislNVN012XXre0Ye64grbHMPDwFtvBYf/d78LjI/b+epqb/gvXszwJ6KMlP5Bv22b91hREbB5s22OixeBt98ODv977wXGxux8VZUFf2D4L13K8CeitJf+TTfTMTICvPOOhb5TAbz1FjA6aucrKrzhv2wZkJPefdhElBmyq+kmXoWFboA7RketjT8w/O+/3yoFACgvB9avDw7/FSsY/kSUsrL7jj5WY2PA3r3B4b97tzUHAUBpqTf8L7kEyM1NbrmJKKPFekfPoI/X+Diwb583/IeG7HxxMdDW5gb/xo3AypVAXnb/EUVEM4dBnwwTE8D+/cHh/+abwIULdr6oCFi3Ljj8W1oY/kQUFwZ9qpiYsCd4A8N/1y5gcNDOz5kDrF0bHP6trUB+fnLLTUQpj0Gfynw+4NAhb/ifP2/nCwq84b96tR0nIvJj0Kcbnw84fDg4/HfuBM6ds/P5+cCaNcHhv2aNjRwioqzEoM8EqsC773rDv6/Pzufn251+4GiftWutOYiIMh6DPlOpAkePesP/zBk7n5dnbfyB4b9unXUEE1FGYdBnE1Xg+HFv+Pf02PncXBvdExj+bW02BJSI0haDPtup2hz9oeHf3W3nc3KAVau84V9amtxyE1HMGPTkpQp0dnrDv6vLzovYQ12B4b9+PVBWltxyE1FYnOuGvESAhgbb/u7v3ONdXW7479wJ/PGPwM9/7r5nxQpv+FdUJOczENGU8Y6ewjt1yr3jd7aODvf8smXBQz03bAAqK5NXXqIsxKYbmnmnT3vD//hx9/ySJd7wr65OXnmJMhyDnmZHb683/I8edc8vWhQc/hs3AnPnJqu0RBmFbfQ0O2pqgOuvt81x9qw3/H/9a/d8c7M3/GtrZ7/sRFmCQU8zr7oauO462xx9fTaTZ2D4P/GEe76x0Rv+dXWzX3aiDMSgp9lRVQW87322Oc6d84b/k0+65xcs8IZ/ff3sl50ozTHoKXkqKoAtW2xznD9vC7gEhv/TT9szAAAwf743/Bcs4CLuRFEw6Cm1lJcDV19tm2Nw0Bv+v/udzfgJWBNP4Dj/jRutKYjhTwSAQU/poLQUuOoq2xwXLgB//Wtw+D/3nBv+tbXe8G9uZvhTVmLQU3oqKQHe8x7bHENDwFtvBYf/P/+zrfIF2LDO0PBftIjhTxlv0qAXkSYADwGYD8AHYIeq3ici1QAeBbAIwFEAn1TVPhERAPcBuAHAEIDPq+quxBSfKEBxMXD55bY5Ll70hv/3vmeLuwPWSRwa/kuWMPwpo0z6wJSI1AOoV9VdIlIGYCeAWwB8HsBZVb1HRL4JoEpVvyEiNwD4KizoNwO4T1U3R/s3+MAUzaqREeDtt4PD/+23gbExO19R4Q3/pUttxk+iFDJjD0ypaheALv/rARHZB6ABwM0AtvgvexDAiwC+4T/+kFoN8qqIVIpIvf/7ECVfYSGwaZNtjpERYM+e4PD/4Q+B0VE7X15uk7kFhv/y5Qx/SgtTaqMXkUUA1gN4DUCdE96q2iUi8/yXNQA4EfC2Dv8xBj2lrsJCu4vfsAH40pfs2OgosHdvcPj/6EdWKQDWSRwa/itW2EIvRCkk5qAXkVIAvwbwNVU9L5HbMMOd8LQPichWAFsBoLm5OdZiEM2eggJbjKWtDbjzTjs2Ngbs2xcc/j/5CTA8bOdLSuz6wPC/5BJb4pEoSWKa1ExE8gE8DeA5Vb3Xf+wAgC3+u/l6AC+q6iUi8hP/64dDr4v0/dlGT2ltfBzYvz84/HfvtlFAgK3XGxr+q1Yx/GnaZmz2Sv8omgdhHa9fCzj+XQBnAjpjq1X1v4vIjQDugtsZ+0NVvSzav8Ggp4wzMQEcOBAc/m++aeP/AWDOHFu0PTD8W1qA/PzklpvSykwG/VUA/h+At2HDKwHg27B2+scANAM4DuBWVT3rrxjuB/Ah2PDKO1Q1aooz6CkrTEwAhw55w39gwM4XFgJr1waHf2urNSERhcH56InSgc8HtLcHh/+uXTbnD2Ahv2ZNcPivXm2VAmU9Bj1RuvL5gHff9YZ/f7+dz8+3sA8M/zVrrDmIsgqDniiTqAJHjgSH/86dNs8/YB27ra3B4b92rXUEU8Zi0BNlOlXg2DFv+J85Y+dzc62DNzD8162zqSIoIzDoibKRKnDihDf8e3rsfE6ODe0MDf/S0uSWm+LCoCciowp0dHjX8e3utvMiwMqVweHf1gaUlSW33DQpBj0RRaYKdHZ6w7/L/1yjiE3nEBj+69fbnD+UMmZsUjMiykAiQEODbTfd5B7v6nJH+ezcCfzpT8AvfuGeX77cG/6VlbNffpoS3tETUXTd3cHhv3On9QM4li4NDv8NG2yef0o4Nt0QUeL09HjD/9gx9/zixd7wnzs3eeXNUAx6Ippdvb0W/IHhf+SIe37hQm/419Ymr7wZgG30RDS7amqA66+3zXH2rDf8H3/cPd/U5A3/urrZL3uGY9ATUeJUVwPXXWebo7/fG/6/+Y17vqHBG/719bNf9gzCoCei2VVZCbzvfbY5zp2zmTwDw//f/92GgQIW9KHhv2ABF3GPEYOeiJKvogLYssU2x8CAN/yfecYN/7o6b/g3NjL8w2DQE1FqKisDrr7aNsfgoK3eFRj+zz5rM34C1rkbGv7NzVkf/gx6IkofpaXAVVfZ5rhwAfjrX4PD/4UXbKEXwDqJN2wIDv9Fi7Iq/Bn0RJTeSkqA97zHNsfwsDf8v/tdW98XsE7i0PBfsiRjw59BT0SZp6gIuPxy2xwXLwJvvRUc/vfeC4yN2fnKSm/4L11qM36mOQY9EWWHOXOAyy6zzTEyArz9dvBTvvfdB4yO2vnycm/4L1+eduHPoCei7FVYCGzaZJtjdBR4553g8L//fqsUAOskXr8+OPxXrLCFXlIUg56IKFBBgYX3hg3usbExYM+e4PD/8Y+tOQiwfoLQ8F+5MmXCn3PdEBHFY2wM2LcvOPx377aOYMCWbGxrCw7/Vatsfd9A27fbFgdOakZENNvGx4H9+4PD/803gaEhO19UZEs3hi7lGGcOM+iJiFLBxATw8svAv/0b8NhjwPnz3msSHPRsoycimgmqNk//nj3A3r22Oa+dxdkB68wtLwdOnnSPOeP3t22LuxknGgY9EdFUqNqqW+EC/cwZ97qKCqC1Fbj5ZqClxV63tNjsnIEPZonEfUcfKwY9EVE4qraGbrhA7+tzr6uqshD/+MeDA72+PmWetGXQE1F2U7VmlHCBfu6ce93cuRbin/pUcKDX1U0v0Ldtm/5nmASDnoiyg6otah4u0AcG3Otqay3EP/vZ4ECvrU3MHXoC2uRDMeiJKLP4fMDx495A37fPpjl21NVZiN9+uxvoq1Zl5Dq2DHoiSk8+H3D0aPhAd8atA9ZW3toKfOELwYE+d27Sij7bGPRElNomJoAjR7yBvn+/+xQqYKNZWluBrVuDA72qKnllTxEMeiJKDePjwLvvhg90Z0IxAGhqshC/9trgQK+oSF7ZU9ykQS8iDwD4CIDTqrraf2w7gC8BcJ4C+Laq/tZ/7lsA7gQwAeC/qupzCSg3EaWrsTHg8GFvoB844E4PDAALF1qIf+ADbqCvXGkPG9GUxHJH/zMA9wN4KOT4D1T1e4EHRKQFwKcBtAJYAOD3IrJCVSdmoKxElE5GR4H2dm+gHzzoLvYBAIsXW4h/+MPBgV5amryyZ5hJg15VXxKRRTF+v5sBPKKqIwCOiEg7gMsA/DnuEhJRahsZAQ4d8gb6oUPu0n0itlRfaytw001uoF9yiU3xSwk1nTb6u0Tk7wG8AeDrqtoHoAHAqwHXdPiPeYjIVgBbAaC5uXkaxSCiWXHxot2NhwZ6e7u7EHdOji2/19oKfPSjwYFeVJTc8mexeIP+xwD+EYD6998H8AUA4Z4mCDuJg6ruALADsNkr4ywHEc204WFrLw8N9MOHbUgjYAtqLFtmIX7rrW6gr1hhS/ZRSokr6FW123ktIj8F8LT/yw4ATQGXNgLojLt0RJQ4Q0M2oiU00N99151kKy/P1khdtw74zGfcQF++3Jbho7QQV9CLSL2qdvm//CiAd/yvnwLwCxG5F9YZuxzAX6ZdSiKK3+Bg+EA/etQN9Px8uxvfsAH43Ocs0FtaLNALCpJafJq+WIZXPgxgC4AaEekAsA3AFhFpgzXLHAXwZQBQ1T0i8hiAvQDGAXyFI26IZsnAgD0VGhrox4651xQUWHv5ZZcBd9zhBvqyZRb2lJG4whRRujl3LnygnzjhXlNYaEMUAyflammxjtLQNUspbXGFKaJ0198fPMOi8zpwZaI5c+yp0GuuccO8pcWGMubmJq/slFIY9ETJdvZs+EDv6nKvKS62QH//+90wb221p0cZ6DQJBj3RbOntDR/o3d3uNSUlFuIf/GBwoDc32xh1ojgw6IlmkrNAdLhAD10guqUFuPHG4EBvbGSg04xj0BPFw1kgOlyghy4Q3dLiLhDtBHroAtFECcSgJ4rGWSA6XKAHLhBdWRm8QLQT6Cm0QDRlLwY9EeAuEB0a6Hv32ugXR3V18ALRTqBPd4FoogRi0FN2cRaIDhfo58+719XWWojfdltwoCdqgWiiBGLQU2ZyFogOF+ihC0S3tNhj/4EPFmXgAtGUvRj0lN6cBaJDA33fPuDCBfe6+noL8DvuCA70LFogmrIXg57Sg7NAdLhAD10guqUF+OIX3UBftcra1omyFIOeUouzQHRooO/fbwtfOJqaLMSvuSY40Csrk1d2ohTFoKfkcBaIDg30AwdsaTrHwoUW4u9/f3Cgc4Foopgx6CmxnAWiwwV66ALRzqP/TqCvXGlPkBLRtDDoaWY4C0SHBvrBg94Fop1H/wMDnQtEEyUMg56mxlkgOjTQDx3yLhDtPPrvBPoll9gsjEQ0qxj0FJ6zQHRooLe3exeIbmmxR/+dQF+xAigqSm75iehvGPTZzlkgOjTQ333XDXRngeg1a+zR/8BA5wLRRCmPQZ8tnAWiQwP9yBHvAtHr1wOf/awb6FwgmiitMegzjbNAdGigHz3qXuMsEH3ppcDtt7uBzgWiiTISgz5dOQtEhwb68ePuNc4C0VdcAdx5pxvoXCCaKKvwtz2VbN9uWyBngejQQO/ocK9xFoh+73vdWRZbWmxsOgOdKOsxBVLF+Dhw993AggXBgd7Z6V7jLBB97bXBgb5oEReIJqKIGPSp4jvfsf2Xvxx8vKEBuOkm27Zs4Th0IpoyUWfERRJt2rRJ33jjjWQXIzm2b7c7+VB1dTb0cWDAPSZii0cvXWrbsmXu66VLbX1SIsoaIrJTVTdNeh2DPoWIuEMdAXvd22sPKR0+bFvg69Ong99fUxO5EuBSd0QZJ9agZ9NNKhOxlY5qa23kTKiBATf0AyuCl18GHnnEfeAJsLlkAoM/sCJoamKnLVEG4293Ktm2bWrXl5UBbW22hRoZsbHzoZXAvn3AM8/YrJKO/Hzr0A1XCSxZYqN6iChtsekmG01MACdPeisB53XgItki1iEcqUmIC30QJQ2bbiiy3Fygudm2a68NPuf0C4SrBJ55BujuDr5+7tzIlcD8+ewXIEoBDHoKFtgvcPnl3vMDAzbhWWgl8Oc/A48+6u0XWLIkfCXQ3Mx+AaJZwt80mpqyMmDdOttCjY4G9ws4lcCBA8Dvfhe8RGBentsvEFoJLFnCaY6JZtCkQS8iDwD4CIDTqrraf6wawKMAFgE4CuCTqtonIgLgPgA3ABgC8HlV3ZWYolPKKSiw2S9XrPCe8/ki9wu8+qrN3RMoWr9AVdXsfB6iDDFpZ6yIXA1gEMBDAUH/HQBnVfUeEfkmgCpV/YaI3ADgq7Cg3wzgPlXdPFkh2Bmb5VSBM2cidw6fOhV8fXV1+Epg2TL2C1BWmbHOWFV9SUQWhRy+GcAW/+sHAbwI4Bv+4w+p1R6vikiliNSralfsRaesI2IPe9XUAJvD3BcMDobvF3jtNeCxx4L7BYqLw/cLLFvGfgHKWvH+X1/nhLeqdonIPP/xBgAnAq7r8B9j0FP8SkuBtWttCzU6Chw75q0EDh0CnnvO1rh15OUBCxeGrwTYL0AZbKZvb8L9zRy2bUhEtgLYCgDNzc0zXAzKGgUFtgLW8uXecz6fzf4Zrjnotde8/QILFoTvE1i2jP0ClNbiDfpup0lGROoBOJOudABoCriuEUCn590AVHUHgB2AtdHHWQ6iyHJybBK4xkbgmmuCz6kCZ8+G7xd49lmgK+SP0KqqyJVAfT37BSilxRv0TwG4HcA9/v2TAcfvEpFHYJ2x59g+TylJxB72mjsXuOwy7/kLF8L3C/zlL8Avf2lPFzuKiqzpJ1wl0NzM5Rkp6WIZXvkwrOO1RkQ6AGyDBfxjInIngOMAbvVf/lvYiJt22PDKOxJQZqLEKykB1qyxLdTYWPh+gfZ2b79Abq71C4SrBJYs4foCNCs41w3RTPL5rNknXL9Ae7stDRmovj585/DSpTaMlCgKznVDlAw5OfawV0MDcPXV3vOB/QKBlcDzzwcvGwnYhHGRKoH6evu3iGLAoCeaTdXVtl16qffc0FD4foE33gB+9avgfoE5cyL3CyxcyH4BCsKgJ0oVxcXA6tW2hRobA44fD//XwAsvAMPD7rXO7KSR+gVKSmbvM1FKYNATpYP8fDewQ6lG7hd49FGgry/4+vnzw1cCTr8Ah4pmHHbGEmW6vr7wlcDhwzbRXKCKisj9AgsWsF8gxbAzlohMVRWwaZNtoYaGgCNHvJXArl3A448D4+PutXPmAIsXh68EFi60p5QpJTHoibJZcTHQ2mpbqPFxt18g9C+BP/zBKglHTo63XyDwNfsFkopBT0Th5eVZ5+2SJcAHPhB8TtWmjw7XJPTLX9ow0kB1dZErgblz2S+QYAx6Ipo6ERvLX18PXHWV93x/f/hK4D/+A3jooeBry8sjdw43NLBfYAYw6Ilo5lVWAhs32hZqeDh8v8Du3cATTwT3CxQWuusLhFYCixZF7xfYvt024qgbIkoh4+PAiRPh+wXa2739Ak1NkZuEysqsiSmDxTrqhkFPROlBFejuBvbvB156CXjxReCVV4IXnXcUF1ulkAL5lkgcXklE6UcVOH3a7uqPH7d94Ovjx+3hsNAAr6qyu/vhYVtdzLnzdzp5t23L6mYcBj0RzZ7z5yMH+IkTQEeH9w59zhwbutnUBFx/vfva2Tc12XKToUQy/o4+Vgx6IpoZIyMW1OEC3Hl9/nzwe3Jz7Ynb5mab6O1jHwsO8OZmDr+cAQx6IprcxIS1j0cK8BMn7Hyo2loL7KVLgWuv9d6J19fbeP1E2LYtMd83DTHoibKdqs2HE6k5xWlSCRz2CFhziRPcbW3eO/HGRltmMVmyuE0+FIOeKNMNDbmBHSnIL1wIfk9+vgV1U5M9EBUY4M6+ooJNKmmCQU+UzsbHbWWqaO3iZ8543zd/voV1ayvwoQ95Ozjr6vhEagZh0BOlKlWgtzd6u3hnp61TG6iy0r0D37zZ2y7e0GBPnFLWYNATJcvAQPTx4h0dwMWLwe8pLHSD+/3v97aLNzXZE6FEARj0RIkwOuoONYzUrNLfH/yenBwbatjUZHPE3HKLt128pobt4jRlDHqiqfL5bChhtHbx7m7vwzpz51pYL14MXHON9058wYLEDTWkrMb/q4gCqQLnzkVvF+/osMW6A5WUuMF9443edvGmJpt/hSgJGPSUXYaHIz+96ewHB4Pfk5dnHZjNzcAVV4RvF6+qYpMKpSwGPWWOiQmb8Cra3XhPj/d9dXUW1qtWBc+l4gR5XZ09qk+Uphj0lB5UbTx4tFEqnZ0W9oHKy93gvvRS7514YyOHGlLGY9BTahgcnPzpzeHh4PcUFLihHTqPirMvL0/O5yFKIQx6SryxMeDkyeijVPr6gt8j4g41XLcOuOkmbwdnbS2f3iSKAYOewot1vU2fz9q9o7WLh1soorraDe4rr/S2iy9YYPOtENG0cSlBCs9ZtOHcucmf3hwdDX5vUVH4xSECX5eUJOdzEWUQLiVI8Tt40PYVFd6FIgAbbrhunc2j8olPeEO9uppDDYlSyLSCXkSOAhgAMAFgXFU3iUg1gEcBLAJwFMAnVbUv0vegFLJ9O3D33e7X4UIesBkTd+60tTnr620mxGh7rhBElFTTarrxB/0mVe0NOPYdAGdV9R4R+SaAKlX9RrTvw6abFBS43ubEhLXDnzpl7e2R9l1d7qLMgfLzbSz6ZJXC/Pkc6kg0BclsurkZwBb/6wcBvAggatBTisvNdYO4rS36tQMD0SuEY8eA116ziiPcTUZVVWx/JVRW8q8EohhNN+gVwPMiogB+oqo7ANSpahcAqGqXiMybbiEpCeJdb7OszLbly6NfNzZmYR/tL4RXXrF96FS9gN35O5VPtAph3jwbb0+UxabbdLNAVTv9Yf4CgK8CeEpVKwOu6VPVqjDv3QpgKwA0NzdvPHbsWNzloAyman0FkSqEwNe9veG/R03N5BXC/Pn2cBX/SqA0MitNN6ra6d+fFpEnAFwGoFtE6v138/UATkd47w4AOwBro59OOSiDidjon4oKYOXK6NeOjgKnT0f/K+HgQduHDgkFbFhoLBXCvHmcTpjSStz/t4pICYAcVR3wv74ewP8C8BSA2wHc498/ORMFJZpUQYHNXdPYGP06VVv0I1qFsH8/8Mc/ep/YBazyqa2NrS+htDQxn5VoCqZzW1IH4AmxP3XzAPxCVZ8VkdcBPCYidwI4DuDW6ReTaAaJWKdvVRXQ0hL92pERbxNR6H7PHtuPj3vfX1ISW4VQU8MZMilh+GQs0Uzw+YCzZycfgnrqlD1tHCo315qEYmk64gIm5McnY4lmU06O3ZXX1ACrV0e/dmjIlhqMViHs3m3XhE67DFincSwVwty5nPSNADDoiWZfcbGtG7t4cfTrJiZsDv5oFcKuXbYPXRULsA7jyR5Uq6+3a+bMScxnpZTAoCdKVU5zzrx5NrdQNIOD0fsSOjqA11+3UUmRHlSL5a+EeJZMjHUmVEoYttETZZPx8dinswhd6AWwkU2xVAh1de6DaoHTadCMYhs9EXnl5blNNuvXR75OdfLpLNrbgd/9LvwzCYD1EdTXJ+Zz0JQw6InIDA3Z08U9PbYPfB267+21/gOfL/z3ys+382fO2NdOc8+2bWzGSQIGPVEmmpiw4Z5TCe5wM48CwSOKamvt2YPa2uBjgfuaGnvK2MGmm6Rj0BOlOlUL4XDhHOnY2bORw7W01A3l+fNtOGi04K6s5DDNNMegJ5pt4+MWxLHeaff0hJ/BE7CROYGhvGZN5MCurbV289keShnvTKg0Yxj0RNOhakMbY73T7u21+XMi3W2Xl7uhvGCBDauMFtwVFak/4ybb5JOOQU8UaGzMOhBjvdPu7bX5cMLJzw8O6ba26E0kc+dyhS1KCAY9ZS5niGCsd9o9PTarZSQVFW4oNzXZ8MRowc357SlFMOgpeab6xOToqHu3Hcuddm9v5DHeBQXBobxx4+Rt2/n5M/GpiWYdn4yl5FC1kRyHDsV2p93bG37WR0dVVeShfuGOlZXxbpvSHp+MpdT2kY/YfrK1ZQF3DdoVK2xbvtwWF3GCu7qad9tEUTDoaXZt3w7cfbf3+KJFNl67r8+28+fdcwMDNkvjrl3usTlz3MVDproFPsxDlAXYdEPJE+2JyfFxa6pxgn8qW7QmHsBGtkynkmCTD6UINt1QesvLsw7QuXOn/t6JCW8lcfZs5Iqhs9OWA4ylkigoiL+SKC5mJUFJwaCn5EnUE5O5udZuX1099feGqySibadOAfv2uZVEtL+Q8/PjryRKSlhJUNwY9JQ8qfjE5HQqCZ9vapXE6dPAgQP2ur9/8kqisnLqFUR19dQrCS4UknHYRk+UCnw+64COp0+ivz/ydMGANYPFWjHU1ABXX83ZJtME2+iJ0klOjt2xV1ZOvpYsYME+OGiVQ38/cPIkcOwYcPSo7Z2ts9NdVaqnJ+Efg1ITg55oNvnbyY5cAAAJAUlEQVR8Nlz0/Png7dy52I45xwcGYvv3SkttKgZnq6gI/to59uKLwNNPu+/jQiEZhUFP6W222pMnJsIH9FRDOtaALivzhnFjY/iQDj3mHC8ttT6HWHz96+5rLhSScdhGT+ltslByAjra3XEsxwYHYytLaEBPFsbhjpeVJXehDwZ92mAbPWW+xx+3/R13RA7pCxcm/z5OQAcGb1UVsHDh1EK6tDSlV2JSVVwYnUBp4SS/9lwoJOPwjp7ST6RpFKZq3jxb3KOuzoK+tDR4KymJ7ViqzrPjb9ZSVbx0qBfff/4AmquLcf9tG5JdMpohsd7RM+gpbY1P+JCXl4v3/sNvcLH/PK5tLMadbfNwSalYU4uzXbgQ/HUsxyJNbxxOQUHslUKsFUhJyfT/OhDBK+09+MELB/H60T40VBbha9ctx62bmqb3fSllsOmGMl5ergXhs//zRjz052P4yUuH8diL5/G+lfPwpfeuwuVLqiHxPk06OuoGfzwVxeCgDXkMPR5tvHuooqK4KoqJ4hK8fiEHlwO47aevoa68EP94y2p8alMTCvJSt2mJEod39JTeAkbdDI6M42cvH8EDLx/F2QujWN1Qji9etQQ3rq1Hvr9SmJVROqpWUQwNudvwsAX+2bP2RGxPj+0DXzv7SEsTTgeHSWYkNt1Q1ro4NoEn3jyJf/3PI2g/PYj55XPwyU2N+MSGRjTXllrYOuEbGMahwTzVc4HHp3Ln7sjPt4nPnK2oKPjrCMe7x3Pw6qlhvHpqGH2ah/nzq3HdVS246r/cyNEzGY5BT1nP51P86VAPHnzlKI6+9hb+sOPLyNUZCOAphPCkx51zRUVT6tQ9MziCZ/ecwq92duDN4/0oyMvBDavn47bNC3HpoiprsuIwyYyX9DZ6EfkQgPsA5AL4v6p6T6L+LaJwcnIE114yD9c+/C/AT7yjdN7cuAWjn74Na5bPR3Fl+YwEcCL1XRjF83tP4em3uvDK4TOY8CmWzSvF/7hxFT6+oRFVJQXBb+AwSfJLyB29iOQCOAjgAwA6ALwO4DOqujfc9byjp1kjglcP9+K3b3fh2XdO4fTACApyc7B5STWuXFaD9yydi9YFFcjNSf6UwBM+xVsd/fjTwR786WAP/nqiHz4FFs4txkfW1uPGNQuwqr4s/g5nSntJbboRkSsAbFfVD/q//hYAqOr/Dnc9g55mTUBzhs+nePNEP559pwsvHujBodP29GtFUT42L67GuqZKrG2swJqGClQWF0T7rtOmqugdHMWeznPYdawPO4/3YffxflwYnYAIsK6xEtesqMV1q+qwuqGc4U4Akt900wDgRMDXHQA2J+jfIopdQHNGTo5g48IqbFxYhX+4ETg9cBF/PnwGL7f34i9HzuL5vd1/u7ahsghLakuwpKYEi2tK0FBVjNqyQswrK0RNaeGkwxZVFUOjEzg3PIbu8xfR2X8RJ/uH0NE3jEPdgzjQPYCzF2zsfo4AK+eX42MbGnHp4mq8d1mNt1mGaAoSFfThbjeC/nQQka0AtgJAc3NzgopBFCLKEMN5ZXNwc1sDbm5rAACcGxrDO53n8NeOfhw8NYB3ey/g17tOYnBk3PPegrwcFBfkojg/F4X5ufCpYsKn8PkUF8d9OD88hnGf96/nssI8LJ1Xiutb6rCirgwr55dhbVPl5NMUEE1Bov5v6gAQ+PhdI4DOwAtUdQeAHYA13SSoHERxqyjOx5XLanDlspq/HVNV9AyO4NS5i+gZGMHpgRH0DoxgcHQcw6MTuDAygZHxCeTmCHJFkJMjKMzLQUVRPsqL8lFRlI/a0kI0VBWhoaoI5XNSo6OXMluigv51AMtFZDGAkwA+DeC2BP1bRLNGRDCvbA7mlc1JdlGIYpaQoFfVcRG5C8BzsOGVD6jqnkT8W0REFF3CGgJV9bcAfpuo709ERLHhDEdERBmOQU9ElOEY9EREGY5BT0SU4Rj0REQZjkFPRJThUmI+ehHpAXAs2eWIUw2A3mQXIgH4udJHJn4mIDM/10x/poWqWjvZRSkR9OlMRN6IZfa4dMPPlT4y8TMBmfm5kvWZ2HRDRJThGPRERBmOQT99O5JdgATh50ofmfiZgMz8XEn5TGyjJyLKcLyjJyLKcAz6OInIrSKyR0R8IrIp5Ny3RKRdRA6IyAeTVcbpEpHtInJSRHb7txuSXaZ4iciH/D+PdhH5ZrLLM1NE5KiIvO3/+aTtwssi8oCInBaRdwKOVYvICyJyyL+vSmYZpyrCZ0rK7xSDPn7vAPgYgJcCD4pIC2yhlVYAHwLwLyKSO/vFmzE/UNU2/5aW0077//v/CMCHAbQA+Iz/55QprvX/fNJ5KOLPYL8vgb4J4A+quhzAH/xfp5OfwfuZgCT8TjHo46Sq+1T1QJhTNwN4RFVHVPUIgHYAl81u6SjEZQDaVfVdVR0F8Ajs50QpQlVfAnA25PDNAB70v34QwC2zWqhpivCZkoJBP/MaAJwI+LrDfyxd3SUib/n/DE2rP50DZNrPJJACeF5EdorI1mQXZobVqWoXAPj385Jcnpky679TDPooROT3IvJOmC3a3aCEOZayQ5sm+Yw/BrAUQBuALgDfT2ph45dWP5MpulJVN8Capb4iIlcnu0AUVVJ+pxK2lGAmUNXr4nhbB4CmgK8bAXTOTIlmXqyfUUR+CuDpBBcnUdLqZzIVqtrp358WkSdgzVQvRX9X2ugWkXpV7RKRegCnk12g6VLVbuf1bP5O8Y5+5j0F4NMiUigiiwEsB/CXJJcpLv5fLsdHYR3Q6eh1AMtFZLGIFMA6y59KcpmmTURKRKTMeQ3geqTvzyicpwDc7n99O4Ank1iWGZGs3yne0cdJRD4K4P8AqAXwjIjsVtUPquoeEXkMwF4A4wC+oqoTySzrNHxHRNpgzRxHAXw5ucWJj6qOi8hdAJ4DkAvgAVXdk+RizYQ6AE+ICGC/y79Q1WeTW6T4iMjDALYAqBGRDgDbANwD4DERuRPAcQC3Jq+EUxfhM21Jxu8Un4wlIspwbLohIspwDHoiogzHoCciynAMeiKiDMegJyLKcAx6IqIMx6AnIspwDHoiogz3/wHHR0P6Q0EpJQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "theta_history = []\n",
    "# 设置允许循环的最大次数为10\n",
    "gradient_descent(0.0, 1.1, n_iters=10)\n",
    "plot_theta_history()\n",
    "print(len(theta_history))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 在线性回归模型中使用梯度下降法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(666)\n",
    "x = 2 * np.random.random(size=100)\n",
    "y = x * 3. + 4. + np.random.normal(size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = x.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 1)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGnFJREFUeJzt3X+sXGWdx/HPl7ZKi6wtS92Fi1VITBtZXAo3G6QbFVCLKNLFTdRIgopp3B9G0O1uCYngJhuasInsxs2arrJqJFh+2cVfC6zFmEWLubUtpUIFUZGLK1WortKVS/nuH/dcOnd6zsyZM+fHc57zfiVN7z1zZua5Z2a+85zv832eY+4uAED7HdV0AwAA5SCgA0AkCOgAEAkCOgBEgoAOAJEgoANAJIYGdDO7wcyeNLMHerZdZ2YPmdn9ZvYlM1tabTMBAMPk6aF/VtL5fdvulvRH7v4aST+QdGXJ7QIAjGhoQHf3b0l6qm/bXe7+XPLrdkknVdA2AMAIFpbwGO+XtCXrRjNbL2m9JB1zzDFnrlq1qoSnBIDu2LFjxy/cffmw/cYK6GZ2laTnJN2YtY+7b5a0WZImJyd9ampqnKcEgM4xs5/k2a9wQDezSyW9TdJ5zoIwANC4QgHdzM6X9HeSXu/uz5TbJABAEXnKFm+S9B1JK83scTO7TNInJR0r6W4z22Vmn6q4nQCAIYb20N393SmbP1NBWwAAY2CmKABEgoAOAJEoow4dQMtt3Tmt6+7cpycOHNSJSxdrw9qVWrd6oulmYUQEdKDjtu6c1pW379HBmUOSpOkDB3Xl7XskiaDeMqRcgI677s59LwTzOQdnDum6O/c11CIURUAHOu6JAwdH2o5wEdCBjjtx6eKRtiNcBHSg4zasXanFixbM27Z40QJtWLuyoRahKAZFgY6bG/ikyqX9COgAtG71BAE8AqRcACASBHQAiAQBHQAiQUAHgEgQ0AEgEgR0AIgEAR0AIkFAB4BIMLEIQJS6uMY7AR1AdLq6xjspFwDR6eoa7wR0ANHp6hrvBHQA0enqGu8EdADR6eoa7wyKAmiFUapWurrGOwEdQPCKVK3UtcZ7SOWRBHQAwRtUtVJl8BwWrEMrjySHDiB4TVStzAXr6QMH5TocrLfunH5hn9DKIwnoAILXRNVKnmAdWnkkAR1A8JqoWskTrEMrjySgAwjeutUTuvbi0zSxdLFM0sTSxbr24tMqzVPnCdahlUcyKAqgFeqqWpmzYe3KeQOe0pHBOrTySAI6AKTIG6yzvmiaKGckoAOIXtHgWvSsoKlyRnLoAKKWVn54+ZZdOv3jd80rQSxTU+WMQwO6md1gZk+a2QM9244zs7vN7OHk/2WVthIACkoLrpJ04ODMEXXlZWmqnDFPD/2zks7v27ZR0jfc/VWSvpH8DgDBGRREq+o1N1XOODSgu/u3JD3Vt/kiSZ9Lfv6cpHUltwsASjEsiFbRa26qnLFoDv0P3P1nkpT8/7KsHc1svZlNmdnU/v37Cz4dABSTFlx7VdFrbqJuXqqhysXdN0vaLEmTk5Ne9fMBQK+5IPrxL+/V08/MzLutyl5z3XXzUvEe+s/N7ARJSv5/srwmAUC51q2e0M6PvVnXv/P02nvNdSraQ79D0qWSNiX//0dpLQKAijTRa65TnrLFmyR9R9JKM3vczC7TbCB/k5k9LOlNye8AgAYN7aG7+7szbjqv5LYAAMbA1H8AGFFIl53rRUAHCgj1A43DqnqNQrvsXC8COjCikD/QXTIoYFf5GjV1fdM8WJwLGFFo15HsomHX+6zyNQrtsnO9COjAiEL+QHfFsIBd1Wu0dee0jjJLva2py871IqADIwrtOpJdNCxgV/EazZ0VHPIjJ7w3edm5XgR0YEShXUeyi4YF7Cpeo2vu2Ju6DO8Cs2BmnBLQgRE1tfASDhsWsMt+jbbunNaBgzOptz3vHsxrT5ULUEDsU8hDl+d6n2W+RoMGU0NKtRHQAbRSnV+qgwZTQ0q1kXIBgCGyeuHLliwK6kyNgA5gbFt3TmvNpm06eeNXtWbTtsouvtyUrJz91Ree2lCL0pFyATDPqFPmuzBzNk/OPgTmKTWVVZmcnPSpqanang+IUZXryPQHZ2m2JzqoQmTNpm2aTskxL1uySDs/9uZS2tV1ZrbD3SeH7UfKBWiRYVPex1VkynzWgOHTz8xEl3oJHSkXoEXKWBiqv4d/zqrluueh/Xoi+ZJIM6jK48Sli1N76JL00Zt364otuwqfSRQ9G+nqapgEdKBF8qxRMuoqhF/Y/tjQ5x1Ua71h7UpdvmVX6m1z0+SL5NWL5Oa37pw+4mLQMeb0s5ByAQKUVTUybMp7kVUIh1l0lA2stV63ekJLFy8a+jijrnY4avpn7m/vDeZFn7utCOhAYAYF5WFT3ouuQjhQ+uKC81zz9lOPaFeaUZ5/1BUTh31ZFV1psU0lmQR0IDDD8uSD1igpugrhIDOHfGjvtr9dC0pYYnbUFROHBewif3vVg9BlI4cOBGZYUB405T1rgLJ3FcL+ssRx2tSrt11Z5Y+jTJNPa+ugxxg0OFt0pcWQr06Uhh46EJiXZuSj8/Qwi6xCeMlZK0rtWWc9z6irHY76GGl/uyQtXbyo8EqLbbuYCT10ICBbd07rt88+d8T2YQOTc8ZdhbCMnnWe56niMaqYzTnsjCc0BHQgINfduU8zh46sBn/J0QtHCmxFg1hbprhnKXsFxlHTPk0joAMByTqVP5BSilcV1no/rG1fcAR0ICChnuLnmXkZ6+zMNn3BMSgKBCTE65Vu3TmtDbfsnle6t+GW3fNK99pW3hcrAjoQkBCvV3rNHXs18/z8vP7M865r7tj7wu9FFvVC+Ui5IFixnsIPE9opftbFkXu3t628L1YEdASpCxdNiMk4uf/Yvrib/HtIuSBInMKHY9mS9IlOvduL5v5jy703/fcQ0BEkTuHDcfWFp2rRgvkzSBctsHnX0yya+8/64r58y67gF8JK03RHhJQLghRq+V4X5a3FLpL7H/QF3cY0W9MdEQI6gtS2GXqxq2qgdtCCWlLYC2GlabojQsoFQQqhfK9N62CHYtRjlrWgVq82pdmankcwVg/dzK6Q9AFJLmmPpPe5+/+V0TCgyfK9mKtsqqrCKHLMetM5WT31NqXZml4qwNyzLgs75I5mE5L+W9Kr3f2gmd0s6Wvu/tms+0xOTvrU1FSh5wPqtGbTttQAM7F0se7deG4DLSpH1mqKZZz9jHvMqmxb25nZDnefHLbfuCmXhZIWm9lCSUskPTHm4wFBaHpwqypVVmGMe8xCSLO1XeGUi7tPm9k/SnpM0kFJd7n7Xf37mdl6SeslacWKFUWfDqhV04NbVanyi6qMYxbaLNm2KdxDN7Nlki6SdLKkEyUdY2aX9O/n7pvdfdLdJ5cvX168pUCNmh7cqsqo1+kcRazHrE3GSbm8UdKP3H2/u89Iul3S2eU0C2hWrKf/owbdUapWYj1mbTJOlctjks4ysyWaTbmcJ4kRT0QjxtP/UaowilatxHbM2mScHPp9ZnarpO9Jek7STkmby2oYgGrkDbptu+I9xqxDd/erJV1dUluAI8S2El/I+o91Vl142yt9YsbUfwQr5sk9oUk71qbZGYP92l7pEzOm/iNYTa9c1yVpx9olWd9+VK2EjYCOYMU6uSdEWcfUJapWWoSUC4IV6+SeEGUd6wVmjFu0CD30iLV9tUAmqtQna9XDQ+6tvoJQ1xDQI9X0pbDKwESV+swd6wXWnzVn3KJNSLlEKpYaYiaq1Gfd6gldsWVX6m2MW7QDAT1SDCgOR437kRi3aDdSLpGqchGmGMSQkqoC4xbtRkCPVB0fzDYPulLjno5xi3Yj5RKpqi+F1fZZnKSksjFu0V4E9IhV+cFs+6AruWLEiIAekToH+ULu4eY5DhvWrky9fmXRlBQDrAgBAT0SdadAQu3h5j0OZaak2p5+QjwI6JEoIwUySi+z7B5uWe0b5TiUlZJqW/qJs4l4EdADk/Zhk4b3JMdNgYzay6x60LVo+5pIBYWUfhoWrDmbiBsBPSBpH7YNt+yWTJo55C9sS/sAjpsCKdLLrLMaIm/7mkgFhZJ+yhOs23Y2gdFQhx6QtA/bzPP+QjCfk1YvPW7deUi9TOnIGve8V89pYmJMKJNx8tTWh/Y6o1z00Gs07HR4lA9V/77jpkBC6WVKo10956WLF2nNpm3z/uZrLz6t1hxx3emnLHmCdUivM8pHQK9JntPhQddx7Jf2ARwnBdLEIGeWQVfP6Q3qi44y/fbZ53Tg4Iykw8f02otP070bz62tvVIYk3HyBOuQXmeUj5RLTfKcDqedui86yrRowfwlTav4AIY05Tvv1XNecvTCXOmorsiT+gnpdUb56KHXJM/pcNape9q2Kj6ATfQy09JQWT3NiaWL5/W8T9741dTHDC0fXFeZYN7UTwhnE6gGAb0meXOXWR+2GD+AWWmod5w5odt2TA9NC7QhH1x3mSDButtIudQklEqIkGSloe55aH+utECdx7ToypKs6og60UOvSdrp8Dmrluu6O/fpii27Ojljb1AaKk9Ps67qknF62ZQJok4E9Br1BqmuzNgblD8uI2VSVophUDvHmYzThrQQ4kHKpSFdOBUfdlWgUNJQw9o5Ti87lL8R3UBAb0gXTsWHfWmFUkI3rJ3jXM4vlL8R3UDKpSFdOBXPW6rZdHAb1s5xJ+OE8DeiG+ihN6QLp+JlX6i6qmuYDmsnvWy0BT30hoSy/keVypxmXuUgcp520stGGxDQGxR7kCjzS6vKZV+78OWKbiCgo1JlfWlVPYgc+5cruqE1AZ3LZnVbFwaRgXG1YlB0WJ0w4teFQWRgXGMFdDNbama3mtlDZvagmb22rIb16sIkHAxGpQkw3Lgpl3+S9J/u/udm9iJJS0po0xG6MAkHw5HnBgYr3EM3s9+T9DpJn5Ekd3/W3Q+U1bBeZdczA0CMxkm5nCJpv6R/N7OdZvZpMzumfyczW29mU2Y2tX///kJPRP4Udahq4hJQl3EC+kJJZ0j6V3dfLem3kjb27+Tum9190t0nly9fXuiJyJ+iagy8Iwbj5NAfl/S4u9+X/H6rUgJ6WcifokpVTlwC6lK4h+7u/yPpp2Y2l/c4T9L3S2kVUDMG3hGDcatcPiTpxqTC5VFJ7xu/SWFhQlM3MHEJMRgroLv7LkmTJbUlOF25qlDblfGlW+ZCYkBTWjP1vwnkVZuVJ1CX9aXLAl2IAQF9APKqzckbqMv80mXgHW3XirVcmsKEpubkXe6BL13gMAL6AExoak7eQM2XLnAYAX0AJjQ1J2+g5ksXOKzzOfRhA2/kVZuRt+qEwUzgsE4HdMoSwzVKoOZLF5gVVUAftR45hrLEmCc+EaiB0UQT0Iv0tttWIdEfvM9ZtVy37ZgO6gwj5i8YIHTRDIoWuapRmyok0lYDvHH7Y7VeyWnY8rKsWAg0K5qAXqS33aYKibQvLM/Yt4ozjDzBmksFAs2KJqAX6W23qSxxlCBdxRlGnmDdthQWEJtocuhFF1dqy8Bb1mqApvk99SJnGHny3nmCNSsWAs2Kpofept52EVnpofectWKsvzlv3jvPGVCbUlhAjKLpoUvt6W0XUdUEmrylm3nOgJjkAzQrqoAeuyq+sPLmvfMG69C/VCmrRMwI6BVqMnjkfe5R8t6hB+thmBmM2EWTQ08zrG666uduqiZ7lOfuUt6bskrELtqA3vQkl6zg8dGbd1fehlECV+yDyb0oq0Tsok25ZAW1a+7YW0uwygoSh9wrP80fNXC1PZWSF2WViF20PfSs4HXg4EwtvfRBQaLq0/w2LWlQpy6ll9BN0Qb0QcGrjpxpWvDoVeVpPoErXZfSS+imaFMuG9au1OVbdqXeVkfOdC5IfPTm3TrkR666MugLZ9zqGOrBs3UlvYRuijagr1s9oY9/ea+efmbmiNvmgmnVZYVzjzXKkgRlldYRuIDuiTblIklXX3hqZuqhriqYUU/zKa0DUFS0PXRpcOphzaZttV2taJTeMqV1AIqKOqBL2cE01MBJaR2AoqJOuQwSamkfFSoAiupsQA81cFJaB6Co1qRcyq5ICbm0jwoVAEW0IqBXtUoegRMSS+oiHq0I6HkvwhCDJoJLlwMaS+oiJq3IoYdakVK2JlaIbHpVyqZR94+YtCKgh1qRUrYmgkvXA1pXOgvohlakXLKuZ3nOquVas2lbbamCqlMTTQSXrgc06v4Rk7F76Ga2wMx2mtlXymhQmt5SPklaYKaDM4d04/bHaksV1JGaaOJMpCtnP1lCLV8Fiigj5fJhSQ+W8DgDrVs98cKHb271wv41DKtMFdSRmmgiuHQ9oFH3j5iMlXIxs5MkvVXSP0j6SCktGiAtqParKlWQ9bhpp+tFNVEbH3I9fl0oX0Usxs2hXy/pbyUdm7WDma2XtF6SVqxYMdaT5QnWVaUKsnKtptl0TFkBoYngQkAD4lA45WJmb5P0pLvvGLSfu29290l3n1y+fHnRp5M0PFhXmSrYsHalLGW7q54rII1r685prdm0TSdv/KrWbNrWmbJEoEvGyaGvkfR2M/uxpC9KOtfMvlBKqzKk5XvngmzVuc91qyeOyNnPCb0ipOu15kBXFE65uPuVkq6UJDN7g6S/cfdLSmpXqqbzvRMtLXHr0kxboMtaUYfeq858b3/d+Tmrluu2HdO5LycXiq7XmgNdUcpMUXf/pru/rYzHCkVamuK2HdN6x5kTrStx63qtOdAVrZj634SsNMUXtj8mSfrEO0/XvRvPDT6YS9SaA13RupRLXQalI6pYka/KZQWaHnsAUA8CeoasuvM5ZQ4q1rGEK7XmQPxIuWRIS1P0K2tQsesrHgIoBz30DL1piqyeelmDilShACgDPfQB1q2e0L0bz9X17zy90kFFqlAAlIGAnkPVK/JRhQKgDKRccqpyUJEqFABlIKAHoj+ozw2IEtQB5EVADwRXnwcwLnLogaB0EcC4COiBoHQRwLgI6IGgdBHAuAjogaB0EcC4WjkoWuVCVk2hdBHAuFoX0GOuBmEBLQDjaF3KhWoQAEjXuoBONQgApGtdQKcaBADStS6gUw0CAOlaNyhKNQgApGtdQJeoBgGANK1LuQAA0rWyhx7jxCIAGFfrAnrME4sAYBytS7kwsQgA0rUuoDOxCADStS6gM7EIANK1LqAzsQgA0rVuUJSJRQCQrnUBXWJiEQCkaV3KBQCQjoAOAJEgoANAJAjoABAJAjoARMLcvb4nM9sv6ScF7368pF+U2Jyy0K7R0K7R0K7RxNquV7j78mE71RrQx2FmU+4+2XQ7+tGu0dCu0dCu0XS9XaRcACASBHQAiESbAvrmphuQgXaNhnaNhnaNptPtak0OHQAwWJt66ACAAQjoABCJIAK6mZ1vZvvM7BEz25hy+4vNbEty+31m9sqe265Mtu8zs7U1t+sjZvZ9M7vfzL5hZq/oue2Qme1K/t1Rc7vea2b7e57/Az23XWpmDyf/Lq25XZ/oadMPzOxAz22VHC8zu8HMnjSzBzJuNzP756TN95vZGT23VXmshrXrPUl77jezb5vZH/fc9mMz25Mcq6ma2/UGM/tVz2v1sZ7bBr7+FbdrQ0+bHkjeT8clt1VyvMzs5WZ2j5k9aGZ7zezDKfvU+/5y90b/SVog6YeSTpH0Ikm7Jb26b5+/lPSp5Od3SdqS/PzqZP8XSzo5eZwFNbbrHElLkp//Yq5dye+/afB4vVfSJ1Pue5ykR5P/lyU/L6urXX37f0jSDTUcr9dJOkPSAxm3XyDp65JM0lmS7qv6WOVs19lzzyfpLXPtSn7/saTjGzpeb5D0lXFf/7Lb1bfvhZK2VX28JJ0g6Yzk52Ml/SDls1jr+yuEHvqfSHrE3R9192clfVHSRX37XCTpc8nPt0o6z8ws2f5Fd/+du/9I0iPJ49XSLne/x92fSX7dLumkkp57rHYNsFbS3e7+lLs/LeluSec31K53S7qppOfO5O7fkvTUgF0ukvR5n7Vd0lIzO0HVHquh7XL3byfPK9X33spzvLKM874su111vbd+5u7fS37+X0kPSuq/UEOt768QAvqEpJ/2/P64jjwoL+zj7s9J+pWk38953yrb1esyzX4TzznazKbMbLuZrSupTaO06x3JKd6tZvbyEe9bZbuUpKZOlrStZ3NVx2uYrHZXeaxG1f/eckl3mdkOM1vfQHtea2a7zezrZnZqsi2I42VmSzQbGG/r2Vz58bLZNPBqSff13VTr+yuEKxZZyrb+WsqsffLct6jcj21ml0ialPT6ns0r3P0JMztF0jYz2+PuP6ypXV+WdJO7/87MPqjZs5tzc963ynbNeZekW939UM+2qo7XME28t3Izs3M0G9D/tGfzmuRYvUzS3Wb2UNKDrcP3NLuuyG/M7AJJWyW9SoEcL82mW+51997efKXHy8xeotkvkMvd/df9N6fcpbL3Vwg99Mclvbzn95MkPZG1j5ktlPRSzZ5+5blvle2Smb1R0lWS3u7uv5vb7u5PJP8/Kumbmv32rqVd7v7Lnrb8m6Qz8963ynb1eJf6TokrPF7DZLW7ymOVi5m9RtKnJV3k7r+c295zrJ6U9CWVl2Ycyt1/7e6/SX7+mqRFZna8AjheiUHvrdKPl5kt0mwwv9Hdb0/Zpd73V9kDBQUGFhZqdkDgZB0eTDm1b5+/0vxB0ZuTn0/V/EHRR1XeoGiedq3W7EDQq/q2L5P04uTn4yU9rJIGiHK264Sen/9M0nY/PBDzo6R9y5Kfj6urXcl+KzU7SGV1HK/kMV+p7EG+t2r+oNV3qz5WOdu1QrNjQmf3bT9G0rE9P39b0vk1tusP5147zQbGx5Jjl+v1r6pdye1zHb1j6jheyd/9eUnXD9in1vdXaQd7zANzgWZHiH8o6apk299rttcrSUdLuiV5g39X0ik9970qud8+SW+puV3/JennknYl/+5Itp8taU/ypt4j6bKa23WtpL3J898jaVXPfd+fHMdHJL2vznYlv18jaVPf/So7Xprtrf1M0oxme0WXSfqgpA8mt5ukf0navEfSZE3Hali7Pi3p6Z731lSy/ZTkOO1OXuOram7XX/e8t7ar5wsn7fWvq13JPu/VbJFE7/0qO16aTYO5pPt7XqcLmnx/MfUfACIRQg4dAFACAjoARIKADgCRIKADQCQI6AAQCQI6AESCgA4Akfh/3IBy0SKTn8QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(x, y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用梯度下降法求线性回归模型的参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "目标：使$\\frac{1}{m}\\sum\\limits_{i=1}^{m}(y^{(i)}-\\hat{y}^{(i)})$尽可能小\n",
    "\n",
    "$\n",
    "J(\\theta)=MSE(y, \\hat{y}) \\quad\\quad \n",
    "\\nabla J(\\theta)=\n",
    "\\begin{pmatrix}\n",
    "\\frac{\\partial J}{\\partial \\theta_0}\\\\\n",
    "\\frac{\\partial J}{\\partial \\theta_1}\\\\\n",
    "\\frac{\\partial J}{\\partial \\theta_2}\\\\\n",
    "\\dots\\\\\n",
    "\\frac{\\partial J}{\\partial \\theta_n}\\\\\n",
    "\\end{pmatrix}\n",
    "=\n",
    "\\frac{2}{m}\\cdot\n",
    "\\begin{pmatrix}\n",
    "\\sum\\limits_{i=1}^{m}(X_b^{(i)}\\theta-y^{(i)})\\\\\n",
    "\\sum\\limits_{i=1}^{m}(X_b^{(i)}\\theta-y^{(i)})\\cdot X_1^{(i)}\\\\\n",
    "\\sum\\limits_{i=1}^{m}(X_b^{(i)}\\theta-y^{(i)})\\cdot X_2^{(i)}\\\\\n",
    "\\dots\\\\\n",
    "\\sum\\limits_{i=1}^{m}(X_b^{(i)}\\theta-y^{(i)})\\cdot X_n^{(i)}\\\\\n",
    "\\end{pmatrix}\n",
    "$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 目标函数\n",
    "def J(theta, X_b, y):\n",
    "    try:\n",
    "        return np.sum((y - X_b.dot(theta)) ** 2) / len(y)\n",
    "    except:\n",
    "        return float('inf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对目标函数中的所有未知数求偏导\n",
    "def dJ(theta, X_b, y):\n",
    "    res = np.empty(len(theta))\n",
    "    res[0] = np.sum(X_b.dot(theta) - y)\n",
    "    for i in range(1, len(theta)):\n",
    "        res[i] = (X_b.dot(theta) - y).dot(X_b[:, i])\n",
    "    return res * 2 / len(X_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 梯度下降法求最小值\n",
    "def gradient_descent(X_b, y, initial_theta, eta, n_iters=1e4, epsilon=1e-8):\n",
    "    theta = initial_theta\n",
    "    i_iter = 0\n",
    "    while i_iter < n_iters:\n",
    "        gradient = dJ(theta, X_b, y)\n",
    "        last_theta = theta\n",
    "        \n",
    "        # 矩阵中是没有这种语法的，一个大坑，需要小心\n",
    "        # theta -= eta * gradient\n",
    "        theta = theta - eta * gradient\n",
    "        if abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon:\n",
    "            break\n",
    "        i_iter += 1\n",
    "    return theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 准备训练数据\n",
    "X_b = np.hstack([np.ones((len(x), 1)), x.reshape(-1, 1)])\n",
    "initial_theta = np.zeros(X_b.shape[1])\n",
    "eta = 0.01\n",
    "\n",
    "theta = gradient_descent(X_b, y, initial_theta, eta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4.02145786, 3.00706277])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用自己封装的梯度下降法求线性回归模型的参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from linearRegression.LinearRegression import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.fit_gradient_descent(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.021457858204859"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.00706277])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 向量化求梯度"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\nabla J(\\theta)=\\frac{2}{m}\\cdot X_b^T\\cdot (X_b\\theta-y)$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "def dJ(theta, X_b, y):\n",
    "    return X_b.T.dot(X_b.dot(theta) - y) * 2. / len(X_b)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from linearRegression.LinearRegression import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.fit_gradient_descent(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.021457858204859"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([3.00706277])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 批量梯度下降法和scikit-learn中的波士顿房价数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston = datasets.load_boston()\n",
    "X = boston.data\n",
    "y = boston.target\n",
    "\n",
    "X = X[y < 50]\n",
    "y = y[y < 50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, seed=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from linearRegression.LinearRegression import LinearRegression\n",
    "lin_reg = LinearRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用正规方程法求回归模型，并进行正确率测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 22 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8129794056212895"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time lin_reg.fit_normal(X_train, y_train)\n",
    "lin_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用批量梯度下降法求回归模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CandyWall\\Anaconda3\\lib\\site-packages\\numpy\\core\\fromnumeric.py:86: RuntimeWarning: overflow encountered in reduce\n",
      "  return ufunc.reduce(obj, axis, dtype, out, **passkwargs)\n",
      "D:\\MachineLearning\\linearRegression\\LinearRegression.py:29: RuntimeWarning: overflow encountered in square\n",
      "  return np.sum((y - X_b.dot(theta)) ** 2) / len(y)\n",
      "D:\\MachineLearning\\linearRegression\\LinearRegression.py:51: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  if abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon:\n",
      "D:\\MachineLearning\\linearRegression\\LinearRegression.py:39: RuntimeWarning: overflow encountered in multiply\n",
      "  return X_b.T.dot(X_b.dot(theta) - y) * 2. / len(X_b)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.fit_gradient_descent(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.42362e+01, 0.00000e+00, 1.81000e+01, 0.00000e+00, 6.93000e-01,\n",
       "        6.34300e+00, 1.00000e+02, 1.57410e+00, 2.40000e+01, 6.66000e+02,\n",
       "        2.02000e+01, 3.96900e+02, 2.03200e+01],\n",
       "       [3.67822e+00, 0.00000e+00, 1.81000e+01, 0.00000e+00, 7.70000e-01,\n",
       "        5.36200e+00, 9.62000e+01, 2.10360e+00, 2.40000e+01, 6.66000e+02,\n",
       "        2.02000e+01, 3.80790e+02, 1.01900e+01],\n",
       "       [1.04690e-01, 4.00000e+01, 6.41000e+00, 1.00000e+00, 4.47000e-01,\n",
       "        7.26700e+00, 4.90000e+01, 4.78720e+00, 4.00000e+00, 2.54000e+02,\n",
       "        1.76000e+01, 3.89250e+02, 6.05000e+00],\n",
       "       [1.15172e+00, 0.00000e+00, 8.14000e+00, 0.00000e+00, 5.38000e-01,\n",
       "        5.70100e+00, 9.50000e+01, 3.78720e+00, 4.00000e+00, 3.07000e+02,\n",
       "        2.10000e+01, 3.58770e+02, 1.83500e+01],\n",
       "       [6.58800e-02, 0.00000e+00, 2.46000e+00, 0.00000e+00, 4.88000e-01,\n",
       "        7.76500e+00, 8.33000e+01, 2.74100e+00, 3.00000e+00, 1.93000e+02,\n",
       "        1.78000e+01, 3.95560e+02, 7.56000e+00],\n",
       "       [2.49800e-02, 0.00000e+00, 1.89000e+00, 0.00000e+00, 5.18000e-01,\n",
       "        6.54000e+00, 5.97000e+01, 6.26690e+00, 1.00000e+00, 4.22000e+02,\n",
       "        1.59000e+01, 3.89960e+02, 8.65000e+00],\n",
       "       [7.75223e+00, 0.00000e+00, 1.81000e+01, 0.00000e+00, 7.13000e-01,\n",
       "        6.30100e+00, 8.37000e+01, 2.78310e+00, 2.40000e+01, 6.66000e+02,\n",
       "        2.02000e+01, 2.72210e+02, 1.62300e+01],\n",
       "       [9.88430e-01, 0.00000e+00, 8.14000e+00, 0.00000e+00, 5.38000e-01,\n",
       "        5.81300e+00, 1.00000e+02, 4.09520e+00, 4.00000e+00, 3.07000e+02,\n",
       "        2.10000e+01, 3.94540e+02, 1.98800e+01],\n",
       "       [1.14320e-01, 0.00000e+00, 8.56000e+00, 0.00000e+00, 5.20000e-01,\n",
       "        6.78100e+00, 7.13000e+01, 2.85610e+00, 5.00000e+00, 3.84000e+02,\n",
       "        2.09000e+01, 3.95580e+02, 7.67000e+00],\n",
       "       [5.69175e+00, 0.00000e+00, 1.81000e+01, 0.00000e+00, 5.83000e-01,\n",
       "        6.11400e+00, 7.98000e+01, 3.54590e+00, 2.40000e+01, 6.66000e+02,\n",
       "        2.02000e+01, 3.92680e+02, 1.49800e+01]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[:10, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=red>直接对数据集使用梯度下降法进行训练，Python解释器在运行的时候会打印警告信息，里面有overflow等字样，然后查看训练过后的系数，发现全部都是NAN，原因是：\n",
    "* 训练数据的量纲问题，不同特征的值可能相差好几百倍，\n",
    "* 学习率η的值过大，沿着梯度的方向迈得过大，在10000次循环的过程中，系数没有收敛。</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "为了验证是学习率η过大而导致系数不收敛，这里将学习率调小一些再次进行测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.fit_gradient_descent(X_train, y_train, eta=0.000001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.10245704,  0.11535876, -0.06248791,  0.00207516,  0.00447152,\n",
       "        0.11954208,  0.04684195,  0.03460927, -0.00452122,  0.00324507,\n",
       "        0.1271939 ,  0.04484862, -0.22542441])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.27586818724477224"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "学习率调小后，模型的预测准确率却很低，显然在运算过程系数还没有收敛，而是达到了梯度下降法中的循环的最大次数，退出了循环，我们再尝试将这个最大次数调高点，让算法多运行一会儿，对训练结果进行查看。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 48.4 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time lin_reg.fit_gradient_descent(X_train, y_train, eta=0.000001, n_iters=1e6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7542932581943915"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "经过48秒的运行，预测准确率也是达到了75%，但是跟正规方程法求解的模型的准确度还是有些差距，说明系数还是没有收敛，程序还需要更多次数的循环来到达最小值，可是学习率η太小了导致运行效率太低，系数收敛太慢，故接下来考虑从量纲的角度解决这个学习效率η设置的小，系数收敛的慢，程序运行时间太长的问题。\n",
    "\n",
    "将学习效率η还原成默认值，对训练数据先进行归一化，然后再使用梯度下降法进行训练。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler(copy=True, with_mean=True, with_std=True)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "standardScaler = StandardScaler()\n",
    "standardScaler.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_standard = standardScaler.transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 219 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time lin_reg.fit_gradient_descent(X_train_standard, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8129873310487505"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_standard = standardScaler.transform(X_test)\n",
    "lin_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到经过数据归一化后的梯度下降法的运行速度大大提升，预测的准确度也跟正规方程法求得的回归模型准确度相当。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 梯度下降法的优势"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = 1000\n",
    "n = 10000\n",
    "\n",
    "big_X = np.random.normal(size=(m, n))\n",
    "# 随机生成这个模型的系数的真值\n",
    "real_theta = np.random.uniform(0.0, 100.0, size=n+1)\n",
    "# 根据big_X和系数产生一组big_y\n",
    "big_y = big_X.dot(real_theta[1:]) + real_theta[0] + np.random.normal(0., 10., size=m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用正规方程求解回归模型系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 34.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "big_reg1 = LinearRegression()\n",
    "%time big_reg1.fit_normal(big_X, big_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用梯度下降法求解回归模型系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3.67 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "big_reg2 = LinearRegression()\n",
    "%time big_reg2.fit_gradient_descent(big_X, big_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到在多特征数的情况下，使用梯度下降法求解回归模型的系数的时候比正规方程求解起来运行速度更快"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 随机梯度下降法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Stochastic Gradient Descent：随机梯度下降法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = 100000\n",
    "\n",
    "x = np.random.normal(size=m)\n",
    "X = x.reshape(-1, 1)\n",
    "y = 4. * x + 3. + np.random.normal(0, 3, size=m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用批量梯度下降法求解回归模型系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "def J(theta, X_b, y):\n",
    "    try:\n",
    "        return np.sum((y - X_b.dot(theta)) ** 2) / len(y)\n",
    "    except:\n",
    "        return float('inf')\n",
    "\n",
    "def dJ(theta, X_b, y):\n",
    "    return X_b.T.dot(X_b.dot(theta) - y) * 2. / len(X_b)\n",
    "\n",
    "def gradient_descent(X_b, y, initial_theta, eta=0.01, n_iters=1e4, epsilon=1e-8):\n",
    "    theta = initial_theta\n",
    "    i_iter = 0\n",
    "    while i_iter < n_iters:\n",
    "        gradient = dJ(theta, X_b, y)\n",
    "        last_theta = theta\n",
    "\n",
    "        # 矩阵中是没有这种语法的，一个大坑，需要小心\n",
    "        # theta -= eta * gradient\n",
    "        theta = theta - eta * gradient\n",
    "        if abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon:\n",
    "            break\n",
    "        i_iter += 1\n",
    "    return theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 737 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_b = np.hstack([np.ones((len(X), 1)), X])\n",
    "initial_theta = np.zeros(X_b.shape[1])\n",
    "eta = 0.01\n",
    "theta = gradient_descent(X_b, y, initial_theta, eta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.9999362, 4.0104029])"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用随机梯度下降法求解回归模型系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 随机取一行元素进行\n",
    "def dJ_sgd(theta, X_b_i, y_i):\n",
    "    return X_b_i.T.dot(X_b_i.dot(theta) - y_i) * 2."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "学习率通过公式计算：$\\eta=\\frac{t_0}{t+t_1}，其中t_0和t_1为常数，t为循环次数$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stochastic_gradient_descent(X_b, y, initial_theta, n_iters):\n",
    "    t0 = 5.\n",
    "    t1 = 50.\n",
    "    \n",
    "    def learning_rate(t):\n",
    "        return t0 / (t + t1)\n",
    "    \n",
    "    theta = initial_theta\n",
    "    for i_iter in range(n_iters):\n",
    "        rand_i = np.random.randint(len(X_b))\n",
    "        gradient = dJ_sgd(theta, X_b[rand_i], y[rand_i])\n",
    "        theta = theta - learning_rate(i_iter) * gradient\n",
    "    return theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 302 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_b = np.hstack([np.ones((len(X), 1)), X])\n",
    "initial_theta = np.zeros(X_b.shape[1])\n",
    "theta = stochastic_gradient_descent(X_b, y, initial_theta, n_iters=len(X_b) // 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.96855677, 4.03898067])"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用随机梯度下降法在很少的循环次数就求解出了一个很接近批量梯度下降法结果的系数，而时间却比批量梯度下降法快很多。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用自己封装的随机梯度下降法求解回归模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "from linearRegression.LinearRegression import LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.05 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg = LinearRegression()\n",
    "%time lin_reg.fit_stochastic_gradient_descent(X, y, epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([4.01810812])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.0043277938266906"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lin_reg.intercept_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 随机梯度下降法和scikit-learn中的波士顿房价数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "boston = datasets.load_boston()\n",
    "X = boston.data\n",
    "y = boston.target\n",
    "\n",
    "X = X[y < 50]\n",
    "y = y[y < 50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "from common.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, seed=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "from linearRegression.LinearRegression import LinearRegression\n",
    "lin_reg = LinearRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据进行归一化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler(copy=True, with_mean=True, with_std=True)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "standardScaler = StandardScaler()\n",
    "standardScaler.fit(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_standard = standardScaler.transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用随机梯度下降法求回归模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12.5 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time lin_reg.fit_stochastic_gradient_descent(X_train_standard, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7874288998787285"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_standard = standardScaler.transform(X_test)\n",
    "lin_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "观察回归模型的预测准确度是78.7%，跟正规方程法求得的模型的预测准确率还有差距，说明还没有找到最优解，所以将迭代次数调大一些，再进行训练。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 255 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time lin_reg.fit_stochastic_gradient_descent(X_train_standard, y_train, epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8127493886330507"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_standard = standardScaler.transform(X_test)\n",
    "lin_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "迭代次数调大后，模型的预测准确率已经跟正规方程求得的模型预测准确率已经一致了。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用scikit-learn中的梯度下降法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import SGDRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3.49 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CandyWall\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDRegressor in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8041326881039682"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_reg = SGDRegressor()\n",
    "%time sgd_reg.fit(X_train_standard, y_train)\n",
    "sgd_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 5 ms\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\CandyWall\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\stochastic_gradient.py:183: FutureWarning: max_iter and tol parameters have been added in SGDRegressor in 0.19. If max_iter is set but tol is left unset, the default value for tol in 0.19 and 0.20 will be None (which is equivalent to -infinity, so it has no effect) but will change in 0.21 to 1e-3. Specify tol to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8127996662633307"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_reg = SGDRegressor(max_iter=100)\n",
    "%time sgd_reg.fit(X_train_standard, y_train)\n",
    "sgd_reg.score(X_test_standard, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 验证梯度计算的正确性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(666)\n",
    "X = np.random.random(size=(1000, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10., 11.])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "real_theta = np.arange(1, 12, dtype = float)\n",
    "real_theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_b = np.hstack([np.ones((len(X), 1)), X])\n",
    "y = X_b.dot(real_theta) + np.random.normal(size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 11)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_b.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 损失函数\n",
    "def J(theta, X_b, y):\n",
    "    try:\n",
    "        return np.sum((y - X_b.dot(theta)) ** 2) / len(X_b)\n",
    "    except:\n",
    "        return float('inf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用推导出的数学表达式求损失函数J在theta这一点的梯度\n",
    "def dJ_math(theta, X_b, y):\n",
    "    return X_b.T.dot(X_b.dot(theta) - y) * 2. / len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用模拟的表达式求损失函数J在theta这一点的梯度\n",
    "def dJ_debug(theta, X_b, y, epsilon = 0.01):\n",
    "    res = np.empty(len(theta))\n",
    "    for i in range(len(theta)):\n",
    "        theta_1 = theta.copy()\n",
    "        theta_1[i] += epsilon\n",
    "        theta_2 = theta.copy()\n",
    "        theta_2[i] -= epsilon\n",
    "        \n",
    "        res[i] = (J(theta_1, X_b, y) - J(theta_2, X_b, y)) / (2 * epsilon)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(dJ, X_b, y, initial_theta, eta=0.01, n_iters=1e4, epsilon=1e-8):\n",
    "    theta = initial_theta\n",
    "    i_iter = 0\n",
    "    while i_iter < n_iters:\n",
    "        gradient = dJ(theta, X_b, y)\n",
    "        last_theta = theta\n",
    "\n",
    "        # 矩阵中是没有这种语法的，一个大坑，需要小心\n",
    "        # theta -= eta * gradient\n",
    "        theta = theta - eta * gradient\n",
    "        if abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon:\n",
    "            break\n",
    "        i_iter += 1\n",
    "    return theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "先使用模拟的表达式求梯度的方法求解模型的系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3.84 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 1.1251597 ,  2.05312521,  2.91522497,  4.11895968,  5.05002117,\n",
       "        5.90494046,  6.97383745,  8.00088367,  8.86213468,  9.98608331,\n",
       "       10.90529198])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_b = np.hstack([np.ones((len(X), 1)), X])\n",
    "initial_theta = np.zeros(X_b.shape[1])\n",
    "%time theta = gradient_descent(dJ_debug, X_b, y, initial_theta)\n",
    "theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "再使用推导出的数学表达式求梯度的方法求解模型的系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 639 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 1.1251597 ,  2.05312521,  2.91522497,  4.11895968,  5.05002117,\n",
       "        5.90494046,  6.97383745,  8.00088367,  8.86213468,  9.98608331,\n",
       "       10.90529198])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_b = np.hstack([np.ones((len(X), 1)), X])\n",
    "initial_theta = np.zeros(X_b.shape[1])\n",
    "%time theta = gradient_descent(dJ_math, X_b, y, initial_theta)\n",
    "theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最后可以看到模拟的表达式和我们使用的数学表达式求梯度的方法求解出的模型系数相近且都接近真实的系数，所以，我们的求梯度的数学表达式是正确的。"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "223.448px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
